From 06b4236a5daab958d0572c277f16be2042abf486 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Dec 2025 17:39:09 +0000 Subject: [PATCH 01/49] deps: Bump the all-dependencies group with 9 updates (#62) Bumps Aspire.Hosting.AppHost from 13.0.1 to 13.0.2 Bumps Aspire.Hosting.RabbitMQ from 13.0.1 to 13.0.2 Bumps Aspire.Hosting.Redis from 13.0.1 to 13.0.2 Bumps Aspire.Hosting.SqlServer from 13.0.1 to 13.0.2 Bumps Aspire.RabbitMQ.Client from 13.0.1 to 13.0.2 Bumps AWSSDK.Extensions.NETCore.Setup from 4.0.3.14 to 4.0.3.15 Bumps AWSSDK.S3 from 4.0.13.1 to 4.0.14.1 Bumps AWSSDK.SecurityToken from 4.0.5.1 to 4.0.5.2 Bumps Sentry.Serilog from 6.0.0-preview.2-prerelease to 6.0.0-rc.2-prerelease --- updated-dependencies: - dependency-name: Aspire.Hosting.AppHost dependency-version: 13.0.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: Aspire.Hosting.RabbitMQ dependency-version: 13.0.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: Aspire.Hosting.Redis dependency-version: 13.0.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: Aspire.Hosting.SqlServer dependency-version: 13.0.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: Aspire.RabbitMQ.Client dependency-version: 13.0.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: AWSSDK.Extensions.NETCore.Setup dependency-version: 4.0.3.15 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: AWSSDK.S3 dependency-version: 4.0.14.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: AWSSDK.SecurityToken dependency-version: 4.0.5.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies - dependency-name: Sentry.Serilog dependency-version: 6.0.0-rc.2-prerelease dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-dependencies ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Directory.Packages.props | 1 + 1 file changed, 1 insertion(+) diff --git a/Directory.Packages.props b/Directory.Packages.props index d299cb9..d883f0c 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,6 +3,7 @@ true + From a71a4a86cdec352f01bbe5a42d4eb4ddf7107eaa Mon Sep 17 00:00:00 2001 From: Sam Gibson Date: Tue, 2 Dec 2025 18:15:02 +0000 Subject: [PATCH 02/49] WIP: Refactors deployment workflow for ECS --- .github/workflows/deploy-reusable.yml | 121 ++++++++++++- .github/workflows/deploy.yml | 87 +++------ infra/README.md | 165 ++++++++++++++++++ infra/services.json | 76 ++++++++ infra/task-definitions/api-task-def.json | 51 ++++++ infra/task-definitions/blocking-task-def.json | 51 ++++++ infra/task-definitions/cleanup-task-def.json | 51 ++++++ .../delius-parser-task-def.json | 51 ++++++ infra/task-definitions/filesync-task-def.json | 51 ++++++ infra/task-definitions/import-task-def.json | 51 ++++++ infra/task-definitions/logging-task-def.json | 51 ++++++ .../matching-engine-task-def.json | 51 ++++++ infra/task-definitions/meow-task-def.json | 51 ++++++ .../offloc-cleaner-task-def.json | 51 ++++++ .../offloc-parser-task-def.json | 51 ++++++ .../visualiser-task-def.json} | 4 +- 16 files changed, 939 insertions(+), 75 deletions(-) create mode 100644 infra/README.md create mode 100644 infra/services.json create mode 100644 infra/task-definitions/api-task-def.json create mode 100644 infra/task-definitions/blocking-task-def.json create mode 100644 infra/task-definitions/cleanup-task-def.json create mode 100644 infra/task-definitions/delius-parser-task-def.json create mode 100644 infra/task-definitions/filesync-task-def.json create mode 100644 infra/task-definitions/import-task-def.json create mode 100644 infra/task-definitions/logging-task-def.json create mode 100644 infra/task-definitions/matching-engine-task-def.json create mode 100644 infra/task-definitions/meow-task-def.json create mode 100644 infra/task-definitions/offloc-cleaner-task-def.json create mode 100644 infra/task-definitions/offloc-parser-task-def.json rename infra/{dms-visualiser-task-def.json => task-definitions/visualiser-task-def.json} (95%) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 1e8ff9d..029efa7 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -1,17 +1,124 @@ name: Reusable Deploy on: - workflow_dispatch: + workflow_call: inputs: environment: required: true - type: choice - options: - - preprod - - prod + type: string + description: 'Environment name (preprod or prod)' + aws-region: + required: false + type: string + default: 'eu-west-2' + secrets: + AWS_ROLE_ARN: + required: true + AWS_SECRETS_ARN: + required: true + +permissions: + id-token: write + contents: read jobs: - placeholder: + load-services: runs-on: ubuntu-latest + outputs: + services: ${{ steps.load.outputs.services }} steps: - - run: echo "Triggered from feature branch. This placeholder should not run." \ No newline at end of file + - name: Checkout + uses: actions/checkout@v5 + + - name: Load service mappings + id: load + run: | + services=$(jq -c '.services' infra/services.json) + echo "services=${services}" >> $GITHUB_OUTPUT + echo "Loaded services:" + echo "${services}" | jq '.' + + deploy: + needs: load-services + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + + strategy: + matrix: + service: ${{ fromJson(needs.load-services.outputs.services) }} + fail-fast: false + + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ inputs.aws-region }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.x + + - name: Run tests + run: dotnet test --configuration Release + + - name: Build and Publish Image + id: build-image + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: ${{ inputs.environment }}/${{ matrix.service.name }} + IMAGE_TAG: ${{ github.sha }} + run: | + PROJECT_PATH="src/${{ matrix.service.project }}" + + if [ ! -f "$PROJECT_PATH" ]; then + echo "Project not found: $PROJECT_PATH - Skipping" + echo "skip=true" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Building and publishing: $PROJECT_PATH" + + dotnet publish "$PROJECT_PATH" \ + --configuration Release \ + --target:PublishContainer \ + --property:ContainerRegistry=$REGISTRY \ + --property:ContainerRepository=$REPOSITORY \ + --property:ContainerImageTag=$IMAGE_TAG + + IMAGE="${REGISTRY}/${REPOSITORY}:${IMAGE_TAG}" + echo "IMAGE=${IMAGE}" >> $GITHUB_OUTPUT + echo "skip=false" >> $GITHUB_OUTPUT + echo "Published image: ${IMAGE}" + + - name: Render ECS task definition + if: steps.build-image.outputs.skip != 'true' + id: render-task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1.8.1 + with: + task-definition: ${{ matrix.service.taskDef }} + container-name: ${{ matrix.service.container }} + image: ${{ steps.build-image.outputs.image }} + environment-variables: | + DOTNET_ENVIRONMENT=${{ inputs.environment == 'prod' && 'Production' || 'PreProduction' }} + secrets: | + AzureAd__ClientId=${{ secrets.AWS_SECRETS_ARN }}:Client_ID:: + AzureAd__ClientSecret=${{ secrets.AWS_SECRETS_ARN }}:Client_Secret:: + API__BaseUrl=${{ secrets.AWS_SECRETS_ARN }}:API_Base_URL:: + + - name: Deploy ECS service + if: steps.build-image.outputs.skip != 'true' + uses: aws-actions/amazon-ecs-deploy-task-definition@v2 + with: + task-definition: ${{ steps.render-task-def.outputs.task-definition }} + service: ${{ matrix.service.name }}-service-1 + cluster: ${{ matrix.service.name }}-cluster + wait-for-service-stability: true diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f1d5f8d..c4801e8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -6,71 +6,26 @@ on: - main permissions: - id-token: write # required for OIDC authentication with AWS + id-token: write + contents: read jobs: - deploy: - runs-on: ubuntu-latest - - environment: ${{ github.ref_name == 'main' && 'prod' || 'dev' }} - - env: - REPOSITORY: dms_visualiser - IMAGE_TAG: ${{ github.ref_name == 'main' && 'prod' || 'dev' }}-${{ github.sha }} - - steps: - - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 - with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - aws-region: eu-west-2 - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2.1.1 - - - name: Setup .NET - uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0 - with: - dotnet-version: 10.0.x - - - name: Run tests - run: dotnet test --configuration Release - - - name: Build and Publish Image - id: build-image - run: | - dotnet publish src/Visualiser/Visualiser.csproj \ - --configuration Release \ - --target:PublishContainer \ - --property:ContainerRegistry=${{ steps.login-ecr.outputs.registry }} \ - --property:ContainerRepository=$REPOSITORY \ - --property:ContainerImageTag=$IMAGE_TAG - - IMAGE=${{ steps.login-ecr.outputs.registry }}/${REPOSITORY}:${IMAGE_TAG} - echo "IMAGE=${IMAGE}" >> $GITHUB_OUTPUT - - - name: Render ECS task definition - id: render-task-def - uses: aws-actions/amazon-ecs-render-task-definition@77954e213ba1f9f9cb016b86a1d4f6fcdea0d57e # v1.8.4 - with: - task-definition: infra/dms-visualiser-task-def.json - container-name: dms-visualiser-container - image: ${{ steps.build-image.outputs.image }} - environment-variables: | - VERY_IMPORTANT=DO_NOT_DELETE_ME - DOTNET_ENVIRONMENT=PreProduction - secrets: | - AzureAd__ClientId=${{ secrets.AWS_SECRETS_ARN }}:Client_ID:: - AzureAd__ClientSecret=${{ secrets.AWS_SECRETS_ARN }}:Client_Secret:: - API__BaseUrl=${{ secrets.AWS_SECRETS_ARN }}:API_Base_URL:: - - - name: Deploy ECS service - uses: aws-actions/amazon-ecs-deploy-task-definition@fc8fc60f3a60ffd500fcb13b209c59d221ac8c8c # v2.6.1 - with: - task-definition: ${{ steps.render-task-def.outputs.task-definition }} - service: dms-visualiser-service-1 - cluster: dms-visualiser-cluster \ No newline at end of file + deploy-preprod: + if: github.ref == 'refs/heads/develop' + uses: ./.github/workflows/deploy-reusable.yml + with: + environment: preprod + aws-region: eu-west-2 + secrets: + AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }} + AWS_SECRETS_ARN: ${{ secrets.AWS_SECRETS_ARN }} + + deploy-prod: + if: github.ref == 'refs/heads/main' + uses: ./.github/workflows/deploy-reusable.yml + with: + environment: prod + aws-region: eu-west-2 + secrets: + AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }} + AWS_SECRETS_ARN: ${{ secrets.AWS_SECRETS_ARN }} \ No newline at end of file diff --git a/infra/README.md b/infra/README.md new file mode 100644 index 0000000..a7b312c --- /dev/null +++ b/infra/README.md @@ -0,0 +1,165 @@ +# DMS Infrastructure - AWS Fargate + +This directory contains AWS Fargate ECS task definitions for the DMS (Data Management System). + +## Contents + +## Structure + +``` +infra/ +├── services.json # Service deployment configuration +├── README.md # This file +└── task-definitions/ # ECS task definition files + ├── api-task-def.json + ├── blocking-task-def.json + └── ... +``` + +### Service Mappings +`services.json` - Central configuration mapping services to projects and task definitions. +This file is used by the CI/CD pipeline to automatically deploy all services. + +**Adding a new service:** +1. Create the task definition JSON file in `task-definitions/` (e.g., `my-service-task-def.json`) +2. Add an entry to `services.json`: +```json +{ + "name": "my-service", + "project": "MyService/MyService.csproj", + "taskDef": "infra/task-definitions/my-service-task-def.json", + "container": "my-service-container" +} +``` +3. Push to `develop` or `main` - deployment is automatic! + +### Task Definitions +Each service has its own ECS Fargate task definition JSON file in `task-definitions/`: +- `api-task-def.json` - REST API service (512 CPU, 1024 MB) +- `blocking-task-def.json` - Blocking service (512 CPU, 1024 MB) +- `cleanup-task-def.json` - Cleanup service (256 CPU, 512 MB) +- `delius-parser-task-def.json` - Delius data parser (512 CPU, 1024 MB) +- `filesync-task-def.json` - File synchronization service (512 CPU, 1024 MB) +- `import-task-def.json` - Data import service (512 CPU, 1024 MB) +- `logging-task-def.json` - Logging aggregation (256 CPU, 512 MB) +- `matching-engine-task-def.json` - Matching engine (1024 CPU, 2048 MB) +- `meow-task-def.json` - Meow service (512 CPU, 1024 MB) +- `offloc-cleaner-task-def.json` - Offloc cleaner (512 CPU, 1024 MB) +- `offloc-parser-task-def.json` - Offloc parser (512 CPU, 1024 MB) +- `visualiser-task-def.json` - Visualiser UI (256 CPU, 512 MB) + +## Prerequisites + +1. AWS CLI configured with appropriate credentials +2. .NET 8+ SDK installed +3. ECR repositories created for each service +4. VPC with public and private subnets +5. RDS databases (or equivalent) for data storage +6. RabbitMQ or Amazon MQ for message queuing + +## Building and Publishing Containers + +### Using .NET SDK Container Publishing (Recommended - No Dockerfiles needed!) + +.NET 8+ has built-in container support. Build and push directly to ECR using CLI arguments: + +```bash +# Login to ECR +aws ecr get-login-password --region eu-west-2 | \ + docker login --username AWS --password-stdin {account-id}.dkr.ecr.eu-west-2.amazonaws.com + +# Build and publish - pass container settings as CLI arguments +dotnet publish src/API/API.csproj \ + --configuration Release \ + --target:PublishContainer \ + --property:ContainerRegistry={account-id}.dkr.ecr.eu-west-2.amazonaws.com \ + --property:ContainerRepository=preprod/api \ + --property:ContainerImageTag=abc123 + +dotnet publish src/Visualiser/Visualiser.csproj \ + --configuration Release \ + --target:PublishContainer \ + --property:ContainerRegistry={account-id}.dkr.ecr.eu-west-2.amazonaws.com \ + --property:ContainerRepository=prod/visualiser \ + --property:ContainerImageTag=def456 + +# Repeat for other services: Blocking, Cleanup, Delius.Parser, FileSync, Import, +# Logging, Matching.Engine, Meow, Offloc.Cleaner, Offloc.Parser +``` + +**Note**: +- Dockerfiles are not required when using SDK container publishing +- Pass configuration via CLI args (like the CI/CD pipeline does) rather than hardcoding in .csproj +- Use environment-specific repository names: `preprod/{service}` or `prod/{service}` + +## Configuration Steps + +### 1. Configure Secrets and Environment Variables + +Each service requires configuration for: +- Database connection strings (via AWS Secrets Manager) +- RabbitMQ connection details +- S3 bucket names +- Sentry DSN (optional) + +Add these to the `secrets` array in task definitions: +```json +"secrets": [ + { + "name": "ConnectionStrings__ClusterDb", + "valueFrom": "arn:aws:secretsmanager:eu-west-2:{account-id}:secret:dms/cluster-db" + } +] +``` + +**Note**: Container images are automatically set by the CI/CD pipeline using the `PLACEHOLDER` value. The deployment workflow dynamically injects the correct image URIs at deploy time. + +## Registering Task Definitions + +**Note**: Task definitions are automatically registered by the CI/CD pipeline during deployment. + +For manual registration (if needed): +```bash +aws ecs register-task-definition --cli-input-json file://task-definitions/api-task-def.json +aws ecs register-task-definition --cli-input-json file://task-definitions/blocking-task-def.json +# ... repeat for each service +``` + +## Create ECS Services + +After registering task definitions, create services: +```bash +aws ecs create-service \ + --cluster dms-cluster-dev \ + --service-name dms-api \ + --task-definition dms-api-task \ + --desired-count 2 \ + --launch-type FARGATE \ + --network-configuration "awsvpcConfiguration={subnets=[subnet-xxxxx],securityGroups=[sg-xxxxx],assignPublicIp=DISABLED}" \ + --load-balancers "targetGroupArn=arn:aws:elasticloadbalancing:...,containerName=api-container,containerPort=8080" +``` + +## Service Architecture + +### Public Services (behind ALB) +- **API** - Main REST API endpoint +- **Visualiser** - Web UI for data visualization + +### Internal Services (no load balancer) +- **Blocking** - Handles blocking operations +- **Cleanup** - Data cleanup tasks +- **Delius Parser** - Parses Delius data files +- **FileSync** - Syncs files to/from S3 +- **Import** - Imports data from external sources +- **Logging** - Aggregates and processes logs +- **Matching Engine** - Performs data matching operations +- **Meow** - Message processing service +- **Offloc Cleaner** - Cleans Offloc data +- **Offloc Parser** - Parses Offloc data files + +## Monitoring + +All services log to CloudWatch Logs: +- Log group format: `/ecs/dms-{service-name}` +- Retention: 30 days (configurable in Terraform) +- Container Insights enabled for metrics diff --git a/infra/services.json b/infra/services.json new file mode 100644 index 0000000..138c4e9 --- /dev/null +++ b/infra/services.json @@ -0,0 +1,76 @@ +{ + "services": [ + { + "name": "api", + "project": "API/API.csproj", + "taskDef": "infra/task-definitions/api-task-def.json", + "container": "api-container" + }, + { + "name": "offloc-cleaner", + "project": "Offloc.Cleaner/Offloc.Cleaner.csproj", + "taskDef": "infra/task-definitions/offloc-cleaner-task-def.json", + "container": "offloc-cleaner-container" + }, + { + "name": "matching-engine", + "project": "Matching.Engine/Matching.Engine.csproj", + "taskDef": "infra/task-definitions/matching-engine-task-def.json", + "container": "matching-engine-container" + }, + { + "name": "cleanup", + "project": "Cleanup/Cleanup.csproj", + "taskDef": "infra/task-definitions/cleanup-task-def.json", + "container": "cleanup-container" + }, + { + "name": "import", + "project": "Import/Import.csproj", + "taskDef": "infra/task-definitions/import-task-def.json", + "container": "import-container" + }, + { + "name": "visualiser", + "project": "Visualiser/Visualiser.csproj", + "taskDef": "infra/task-definitions/visualiser-task-def.json", + "container": "visualiser-container" + }, + { + "name": "offloc-parser", + "project": "Offloc.Parser/Offloc.Parser.csproj", + "taskDef": "infra/task-definitions/offloc-parser-task-def.json", + "container": "offloc-parser-container" + }, + { + "name": "delius-parser", + "project": "Delius.Parser/Delius.Parser.csproj", + "taskDef": "infra/task-definitions/delius-parser-task-def.json", + "container": "delius-parser-container" + }, + { + "name": "blocking", + "project": "Blocking/Blocking.csproj", + "taskDef": "infra/task-definitions/blocking-task-def.json", + "container": "blocking-container" + }, + { + "name": "logging", + "project": "Logging/Logging.csproj", + "taskDef": "infra/task-definitions/logging-task-def.json", + "container": "logging-container" + }, + { + "name": "filesync", + "project": "FileSync/FileSync.csproj", + "taskDef": "infra/task-definitions/filesync-task-def.json", + "container": "filesync-container" + }, + { + "name": "meow", + "project": "Meow/Meow.csproj", + "taskDef": "infra/task-definitions/meow-task-def.json", + "container": "meow-container" + } + ] +} diff --git a/infra/task-definitions/api-task-def.json b/infra/task-definitions/api-task-def.json new file mode 100644 index 0000000..1d4d559 --- /dev/null +++ b/infra/task-definitions/api-task-def.json @@ -0,0 +1,51 @@ +{ + "containerDefinitions": [ + { + "name": "api-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "portMappings": [ + { + "containerPort": 8080, + "hostPort": 8080, + "protocol": "tcp", + "name": "http" + }, + { + "containerPort": 8081, + "hostPort": 8081, + "protocol": "tcp", + "name": "https" + } + ], + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-api", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-api-task", + "executionRoleArn": "ecsTaskExecutionRole", + "networkMode": "awsvpc", + "volumes": [], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" +} diff --git a/infra/task-definitions/blocking-task-def.json b/infra/task-definitions/blocking-task-def.json new file mode 100644 index 0000000..6de9856 --- /dev/null +++ b/infra/task-definitions/blocking-task-def.json @@ -0,0 +1,51 @@ +{ + "containerDefinitions": [ + { + "name": "blocking-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ + { + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false + } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-blocking", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-blocking-task", + "executionRoleArn": "ecsTaskExecutionRole", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" +} diff --git a/infra/task-definitions/cleanup-task-def.json b/infra/task-definitions/cleanup-task-def.json new file mode 100644 index 0000000..3ae485f --- /dev/null +++ b/infra/task-definitions/cleanup-task-def.json @@ -0,0 +1,51 @@ +{ + "containerDefinitions": [ + { + "name": "cleanup-container", + "image": "PLACEHOLDER", + "cpu": 256, + "memory": 512, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ + { + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false + } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-cleanup", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-cleanup-task", + "executionRoleArn": "ecsTaskExecutionRole", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "256", + "memory": "512" +} diff --git a/infra/task-definitions/delius-parser-task-def.json b/infra/task-definitions/delius-parser-task-def.json new file mode 100644 index 0000000..45833e0 --- /dev/null +++ b/infra/task-definitions/delius-parser-task-def.json @@ -0,0 +1,51 @@ +{ + "containerDefinitions": [ + { + "name": "delius-parser-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ + { + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false + } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-delius-parser", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-delius-parser-task", + "executionRoleArn": "ecsTaskExecutionRole", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" +} diff --git a/infra/task-definitions/filesync-task-def.json b/infra/task-definitions/filesync-task-def.json new file mode 100644 index 0000000..6f827b6 --- /dev/null +++ b/infra/task-definitions/filesync-task-def.json @@ -0,0 +1,51 @@ +{ + "containerDefinitions": [ + { + "name": "filesync-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ + { + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false + } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-filesync", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-filesync-task", + "executionRoleArn": "ecsTaskExecutionRole", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" +} diff --git a/infra/task-definitions/import-task-def.json b/infra/task-definitions/import-task-def.json new file mode 100644 index 0000000..5fa3fba --- /dev/null +++ b/infra/task-definitions/import-task-def.json @@ -0,0 +1,51 @@ +{ + "containerDefinitions": [ + { + "name": "import-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ + { + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false + } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-import", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-import-task", + "executionRoleArn": "ecsTaskExecutionRole", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" +} diff --git a/infra/task-definitions/logging-task-def.json b/infra/task-definitions/logging-task-def.json new file mode 100644 index 0000000..7ce40e1 --- /dev/null +++ b/infra/task-definitions/logging-task-def.json @@ -0,0 +1,51 @@ +{ + "containerDefinitions": [ + { + "name": "logging-container", + "image": "PLACEHOLDER", + "cpu": 256, + "memory": 512, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ + { + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false + } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-logging", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-logging-task", + "executionRoleArn": "ecsTaskExecutionRole", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "256", + "memory": "512" +} diff --git a/infra/task-definitions/matching-engine-task-def.json b/infra/task-definitions/matching-engine-task-def.json new file mode 100644 index 0000000..630e0d0 --- /dev/null +++ b/infra/task-definitions/matching-engine-task-def.json @@ -0,0 +1,51 @@ +{ + "containerDefinitions": [ + { + "name": "matching-engine-container", + "image": "PLACEHOLDER", + "cpu": 1024, + "memory": 2048, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ + { + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false + } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-matching-engine", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-matching-engine-task", + "executionRoleArn": "ecsTaskExecutionRole", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "1024", + "memory": "2048" +} diff --git a/infra/task-definitions/meow-task-def.json b/infra/task-definitions/meow-task-def.json new file mode 100644 index 0000000..bc0dcf7 --- /dev/null +++ b/infra/task-definitions/meow-task-def.json @@ -0,0 +1,51 @@ +{ + "containerDefinitions": [ + { + "name": "meow-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ + { + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false + } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-meow", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-meow-task", + "executionRoleArn": "ecsTaskExecutionRole", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" +} diff --git a/infra/task-definitions/offloc-cleaner-task-def.json b/infra/task-definitions/offloc-cleaner-task-def.json new file mode 100644 index 0000000..6dfd267 --- /dev/null +++ b/infra/task-definitions/offloc-cleaner-task-def.json @@ -0,0 +1,51 @@ +{ + "containerDefinitions": [ + { + "name": "offloc-cleaner-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ + { + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false + } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-offloc-cleaner", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-offloc-cleaner-task", + "executionRoleArn": "ecsTaskExecutionRole", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" +} diff --git a/infra/task-definitions/offloc-parser-task-def.json b/infra/task-definitions/offloc-parser-task-def.json new file mode 100644 index 0000000..8b7bd83 --- /dev/null +++ b/infra/task-definitions/offloc-parser-task-def.json @@ -0,0 +1,51 @@ +{ + "containerDefinitions": [ + { + "name": "offloc-parser-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ + { + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false + } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-offloc-parser", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-offloc-parser-task", + "executionRoleArn": "ecsTaskExecutionRole", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" +} diff --git a/infra/dms-visualiser-task-def.json b/infra/task-definitions/visualiser-task-def.json similarity index 95% rename from infra/dms-visualiser-task-def.json rename to infra/task-definitions/visualiser-task-def.json index a9f25ac..b0dc1bd 100644 --- a/infra/dms-visualiser-task-def.json +++ b/infra/task-definitions/visualiser-task-def.json @@ -1,7 +1,7 @@ { "containerDefinitions": [ { - "name": "dms-visualiser-container", + "name": "visualiser-container", "image": "PLACEHOLDER", "cpu": 256, "memory": 512, @@ -26,7 +26,7 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "ecs/dms-visualiser", + "awslogs-group": "ecs/visualiser", "awslogs-region": "eu-west-2", "awslogs-stream-prefix": "ecs" } From e40e8c3998af5c9e36ce250f93f15305ed4293c9 Mon Sep 17 00:00:00 2001 From: Sam Gibson Date: Thu, 4 Dec 2025 12:22:13 +0000 Subject: [PATCH 03/49] Configures health check endpoints + request timeouts + output caching This ensures consistent configuration across all services. --- src/API/Program.cs | 5 +++ .../Aspire.ServiceDefaults/Extensions.cs | 35 +++++++++++++------ src/Visualiser/Program.cs | 11 ++++-- src/Visualiser/Visualiser.csproj | 1 + 4 files changed, 39 insertions(+), 13 deletions(-) diff --git a/src/API/Program.cs b/src/API/Program.cs index d818c92..0240dff 100644 --- a/src/API/Program.cs +++ b/src/API/Program.cs @@ -14,6 +14,8 @@ // Add service defaults & Aspire components. builder.AddServiceDefaults(); +builder.Services.AddRequestTimeouts(); +builder.Services.AddOutputCache(); builder.Services.AddOpenApiDocument(options => { @@ -120,6 +122,9 @@ app.UseAuthentication(); app.UseAuthorization(); +app.UseRequestTimeouts(); +app.UseOutputCache(); + app.RegisterClusteringEndpoints() .RegisterDeliusEndpoints() .RegisterOfflocEndpoints() diff --git a/src/Aspire/Aspire.ServiceDefaults/Extensions.cs b/src/Aspire/Aspire.ServiceDefaults/Extensions.cs index 274e4b6..e296cae 100644 --- a/src/Aspire/Aspire.ServiceDefaults/Extensions.cs +++ b/src/Aspire/Aspire.ServiceDefaults/Extensions.cs @@ -86,6 +86,15 @@ private static IHostApplicationBuilder AddOpenTelemetryExporters(this IHostAppli public static IHostApplicationBuilder AddDefaultHealthChecks(this IHostApplicationBuilder builder) { + builder.Services.AddRequestTimeouts( + configure: static timeouts => + timeouts.AddPolicy("HealthChecks", TimeSpan.FromSeconds(5))); + + builder.Services.AddOutputCache( + configureOptions: static caching => + caching.AddPolicy("HealthChecks", + build: static policy => policy.Expire(TimeSpan.FromSeconds(10)))); + builder.Services.AddHealthChecks() // Add a default liveness check to ensure app is responsive .AddCheck("self", () => HealthCheckResult.Healthy(), ["live"]); @@ -97,17 +106,23 @@ public static WebApplication MapDefaultEndpoints(this WebApplication app) { // Adding health checks endpoints to applications in non-development environments has security implications. // See https://aka.ms/dotnet/aspire/healthchecks for details before enabling these endpoints in non-development environments. - if (app.Environment.IsDevelopment()) - { - // All health checks must pass for app to be considered ready to accept traffic after starting - app.MapHealthChecks("/health"); - // Only health checks tagged with the "live" tag must pass for app to be considered alive - app.MapHealthChecks("/alive", new HealthCheckOptions - { - Predicate = r => r.Tags.Contains("live") - }); - } + var healthChecks = app.MapGroup(""); + + healthChecks + .CacheOutput("HealthChecks") + .WithRequestTimeout("HealthChecks"); + + // All health checks must pass for app to be + // considered ready to accept traffic after starting + healthChecks.MapHealthChecks("/health").AllowAnonymous(); + + // Only health checks tagged with the "live" tag + // must pass for app to be considered alive + healthChecks.MapHealthChecks("/alive", new() + { + Predicate = static r => r.Tags.Contains("live") + }).AllowAnonymous(); return app; } diff --git a/src/Visualiser/Program.cs b/src/Visualiser/Program.cs index b325355..c2c238a 100644 --- a/src/Visualiser/Program.cs +++ b/src/Visualiser/Program.cs @@ -8,7 +8,9 @@ builder.UseDmsSerilog(); -builder.Services.AddHealthChecks(); +builder.AddServiceDefaults(); +builder.Services.AddRequestTimeouts(); +builder.Services.AddOutputCache(); builder.Services.AddAuthentication(OpenIdConnectDefaults.AuthenticationScheme) .AddMicrosoftIdentityWebApp(builder.Configuration) @@ -51,14 +53,17 @@ app.UseRouting(); -app.MapHealthChecks("/healthz").AllowAnonymous(); - app.UseAuthentication(); app.UseAuthorization(); +app.UseRequestTimeouts(); +app.UseOutputCache(); + app.MapStaticAssets(); app.MapRazorPages() .WithStaticAssets(); + app.MapControllers(); +app.MapDefaultEndpoints(); app.Run(); diff --git a/src/Visualiser/Visualiser.csproj b/src/Visualiser/Visualiser.csproj index 4671b49..87dd112 100644 --- a/src/Visualiser/Visualiser.csproj +++ b/src/Visualiser/Visualiser.csproj @@ -20,6 +20,7 @@ + From 56e00d3b9287b1cd45ef78962f4913199c2c2316 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Fri, 5 Dec 2025 14:43:07 +0000 Subject: [PATCH 04/49] Adds AWS task and execution role ARNs to deployment workflows and task definitions --- .github/workflows/deploy-reusable.yml | 23 ++++++++++++++++++- .github/workflows/deploy.yml | 6 ++++- infra/task-definitions/api-task-def.json | 1 - infra/task-definitions/blocking-task-def.json | 1 - infra/task-definitions/cleanup-task-def.json | 1 - .../delius-parser-task-def.json | 1 - infra/task-definitions/filesync-task-def.json | 1 - infra/task-definitions/import-task-def.json | 1 - infra/task-definitions/logging-task-def.json | 1 - .../matching-engine-task-def.json | 1 - infra/task-definitions/meow-task-def.json | 1 - .../offloc-cleaner-task-def.json | 1 - .../offloc-parser-task-def.json | 1 - .../task-definitions/visualiser-task-def.json | 4 +--- 14 files changed, 28 insertions(+), 16 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 029efa7..5a0c08d 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -16,6 +16,10 @@ on: required: true AWS_SECRETS_ARN: required: true + AWS_TASK_ROLE_ARN: + required: true + AWS_EXECUTION_ROLE_ARN: + required: true permissions: id-token: write @@ -107,12 +111,29 @@ jobs: task-definition: ${{ matrix.service.taskDef }} container-name: ${{ matrix.service.container }} image: ${{ steps.build-image.outputs.image }} + task-role-arn: ${{ secrets.AWS_TASK_ROLE_ARN }} + execution-role-arn: ${{ secrets.AWS_EXECUTION_ROLE_ARN }} environment-variables: | DOTNET_ENVIRONMENT=${{ inputs.environment == 'prod' && 'Production' || 'PreProduction' }} secrets: | + API__BaseUrl=${{ secrets.AWS_SECRETS_ARN }}:API_Base_URL:: + Authentication__ApiKey=${{ secrets.AWS_SECRETS_ARN }}:Authentication_ApiKey:: + AWS__S3__BucketName=${{ secrets.AWS_SECRETS_ARN }}:S3_BucketName:: AzureAd__ClientId=${{ secrets.AWS_SECRETS_ARN }}:Client_ID:: AzureAd__ClientSecret=${{ secrets.AWS_SECRETS_ARN }}:Client_Secret:: - API__BaseUrl=${{ secrets.AWS_SECRETS_ARN }}:API_Base_URL:: + ConnectionStrings__AuditDb=${{ secrets.AWS_SECRETS_ARN }}:AuditDb:: + ConnectionStrings__ClusterDb=${{ secrets.AWS_SECRETS_ARN }}:ClusterDb:: + ConnectionStrings__DeliusRunningPictureDb=${{ secrets.AWS_SECRETS_ARN }}:DeliusRunningPictureDb:: + ConnectionStrings__DeliusStagingDb=${{ secrets.AWS_SECRETS_ARN }}:DeliusStagingDb:: + ConnectionStrings__MatchingDb=${{ secrets.AWS_SECRETS_ARN }}:MatchingDb:: + ConnectionStrings__OfflocRunningPictureDb=${{ secrets.AWS_SECRETS_ARN }}:OfflocRunningPictureDb:: + ConnectionStrings__OfflocStagingDb=${{ secrets.AWS_SECRETS_ARN }}:OfflocStagingDb:: + ConnectionStrings__CatsRabbitMQ=${{ secrets.AWS_SECRETS_ARN }}:CatsRabbitMQ:: + ConnectionStrings__RabbitMQ=${{ secrets.AWS_SECRETS_ARN }}:RabbitMQ:: + DMSFilesBasePath=${{ secrets.AWS_SECRETS_ARN }}:DMSFilesBasePath:: + Sentry_Dsn=${{ secrets.AWS_SECRETS_ARN }}:Sentry_Dsn:: + + - name: Deploy ECS service if: steps.build-image.outputs.skip != 'true' diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c4801e8..cf01e0c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -19,6 +19,8 @@ jobs: secrets: AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }} AWS_SECRETS_ARN: ${{ secrets.AWS_SECRETS_ARN }} + AWS_TASK_ROLE_ARN: ${{ secrets.AWS_TASK_ROLE_ARN }} + AWS_EXECUTION_ROLE_ARN: ${{ secrets.AWS_EXECUTION_ROLE_ARN }} deploy-prod: if: github.ref == 'refs/heads/main' @@ -28,4 +30,6 @@ jobs: aws-region: eu-west-2 secrets: AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }} - AWS_SECRETS_ARN: ${{ secrets.AWS_SECRETS_ARN }} \ No newline at end of file + AWS_SECRETS_ARN: ${{ secrets.AWS_SECRETS_ARN }} + AWS_TASK_ROLE_ARN: ${{ secrets.AWS_TASK_ROLE_ARN }} + AWS_EXECUTION_ROLE_ARN: ${{ secrets.AWS_EXECUTION_ROLE_ARN }} \ No newline at end of file diff --git a/infra/task-definitions/api-task-def.json b/infra/task-definitions/api-task-def.json index 1d4d559..995e44f 100644 --- a/infra/task-definitions/api-task-def.json +++ b/infra/task-definitions/api-task-def.json @@ -36,7 +36,6 @@ } ], "family": "dms-api-task", - "executionRoleArn": "ecsTaskExecutionRole", "networkMode": "awsvpc", "volumes": [], "placementConstraints": [], diff --git a/infra/task-definitions/blocking-task-def.json b/infra/task-definitions/blocking-task-def.json index 6de9856..1985ece 100644 --- a/infra/task-definitions/blocking-task-def.json +++ b/infra/task-definitions/blocking-task-def.json @@ -28,7 +28,6 @@ } ], "family": "dms-blocking-task", - "executionRoleArn": "ecsTaskExecutionRole", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/cleanup-task-def.json b/infra/task-definitions/cleanup-task-def.json index 3ae485f..8fe6bc4 100644 --- a/infra/task-definitions/cleanup-task-def.json +++ b/infra/task-definitions/cleanup-task-def.json @@ -28,7 +28,6 @@ } ], "family": "dms-cleanup-task", - "executionRoleArn": "ecsTaskExecutionRole", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/delius-parser-task-def.json b/infra/task-definitions/delius-parser-task-def.json index 45833e0..20b3363 100644 --- a/infra/task-definitions/delius-parser-task-def.json +++ b/infra/task-definitions/delius-parser-task-def.json @@ -28,7 +28,6 @@ } ], "family": "dms-delius-parser-task", - "executionRoleArn": "ecsTaskExecutionRole", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/filesync-task-def.json b/infra/task-definitions/filesync-task-def.json index 6f827b6..6e6629e 100644 --- a/infra/task-definitions/filesync-task-def.json +++ b/infra/task-definitions/filesync-task-def.json @@ -28,7 +28,6 @@ } ], "family": "dms-filesync-task", - "executionRoleArn": "ecsTaskExecutionRole", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/import-task-def.json b/infra/task-definitions/import-task-def.json index 5fa3fba..2e253c1 100644 --- a/infra/task-definitions/import-task-def.json +++ b/infra/task-definitions/import-task-def.json @@ -28,7 +28,6 @@ } ], "family": "dms-import-task", - "executionRoleArn": "ecsTaskExecutionRole", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/logging-task-def.json b/infra/task-definitions/logging-task-def.json index 7ce40e1..984471d 100644 --- a/infra/task-definitions/logging-task-def.json +++ b/infra/task-definitions/logging-task-def.json @@ -28,7 +28,6 @@ } ], "family": "dms-logging-task", - "executionRoleArn": "ecsTaskExecutionRole", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/matching-engine-task-def.json b/infra/task-definitions/matching-engine-task-def.json index 630e0d0..5b3398d 100644 --- a/infra/task-definitions/matching-engine-task-def.json +++ b/infra/task-definitions/matching-engine-task-def.json @@ -28,7 +28,6 @@ } ], "family": "dms-matching-engine-task", - "executionRoleArn": "ecsTaskExecutionRole", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/meow-task-def.json b/infra/task-definitions/meow-task-def.json index bc0dcf7..f2d7319 100644 --- a/infra/task-definitions/meow-task-def.json +++ b/infra/task-definitions/meow-task-def.json @@ -28,7 +28,6 @@ } ], "family": "dms-meow-task", - "executionRoleArn": "ecsTaskExecutionRole", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/offloc-cleaner-task-def.json b/infra/task-definitions/offloc-cleaner-task-def.json index 6dfd267..2811bb8 100644 --- a/infra/task-definitions/offloc-cleaner-task-def.json +++ b/infra/task-definitions/offloc-cleaner-task-def.json @@ -28,7 +28,6 @@ } ], "family": "dms-offloc-cleaner-task", - "executionRoleArn": "ecsTaskExecutionRole", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/offloc-parser-task-def.json b/infra/task-definitions/offloc-parser-task-def.json index 8b7bd83..faa3b5e 100644 --- a/infra/task-definitions/offloc-parser-task-def.json +++ b/infra/task-definitions/offloc-parser-task-def.json @@ -28,7 +28,6 @@ } ], "family": "dms-offloc-parser-task", - "executionRoleArn": "ecsTaskExecutionRole", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/visualiser-task-def.json b/infra/task-definitions/visualiser-task-def.json index b0dc1bd..b857867 100644 --- a/infra/task-definitions/visualiser-task-def.json +++ b/infra/task-definitions/visualiser-task-def.json @@ -34,9 +34,7 @@ "systemControls": [] } ], - "family": "dms-visualiser-task", - "taskRoleArn": "arn:aws:iam::035941410605:role/DMS-PreProd_ecs_task_role", - "executionRoleArn": "ecsTaskExecutionRole", + "family": "visualiser-task", "networkMode": "awsvpc", "revision": 6, "volumes": [], From 9cf3a14bd4aa45848988f2b3322eed73b4919ca2 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Wed, 11 Feb 2026 12:08:55 +0000 Subject: [PATCH 05/49] Fetch IAM Role ARNs from Secrets Manager and update ECS task definition references --- .github/workflows/deploy-reusable.yml | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 5a0c08d..fc66154 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -16,10 +16,6 @@ on: required: true AWS_SECRETS_ARN: required: true - AWS_TASK_ROLE_ARN: - required: true - AWS_EXECUTION_ROLE_ARN: - required: true permissions: id-token: write @@ -66,6 +62,24 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v2 + - name: Fetch IAM Role ARNs from Secrets Manager + id: fetch-roles + run: | + SECRET_JSON=$(aws secretsmanager get-secret-value \ + --secret-id ${{ secrets.AWS_SECRETS_ARN }} \ + --query 'SecretString' \ + --output text) + + TASK_ROLE=$(echo "$SECRET_JSON" | jq -r '.Task_Role_ARN') + EXEC_ROLE=$(echo "$SECRET_JSON" | jq -r '.Execution_Role_ARN') + + # Mask secrets in logs + echo "::add-mask::$TASK_ROLE" + echo "::add-mask::$EXEC_ROLE" + + echo "task-role=${TASK_ROLE}" >> $GITHUB_OUTPUT + echo "exec-role=${EXEC_ROLE}" >> $GITHUB_OUTPUT + - name: Setup .NET uses: actions/setup-dotnet@v5 with: @@ -111,8 +125,8 @@ jobs: task-definition: ${{ matrix.service.taskDef }} container-name: ${{ matrix.service.container }} image: ${{ steps.build-image.outputs.image }} - task-role-arn: ${{ secrets.AWS_TASK_ROLE_ARN }} - execution-role-arn: ${{ secrets.AWS_EXECUTION_ROLE_ARN }} + task-role-arn: ${{ steps.fetch-roles.outputs.task-role }} + execution-role-arn: ${{ steps.fetch-roles.outputs.exec-role }} environment-variables: | DOTNET_ENVIRONMENT=${{ inputs.environment == 'prod' && 'Production' || 'PreProduction' }} secrets: | From bec12471eda33b671f62c1e6d3283e347f2e5b20 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Wed, 11 Feb 2026 14:00:02 +0000 Subject: [PATCH 06/49] Add workflow_dispatch inputs for environment and AWS region in deploy workflow --- .github/workflows/deploy-reusable.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index fc66154..cb8db72 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -16,6 +16,20 @@ on: required: true AWS_SECRETS_ARN: required: true + workflow_dispatch: + inputs: + environment: + required: true + type: choice + description: 'Environment to deploy to' + options: + - preprod + - prod + aws-region: + required: false + type: string + default: 'eu-west-2' + description: 'AWS region' permissions: id-token: write From 0de6438aacf34d9d48d65e1be0e5f5197a5a7ccb Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Wed, 11 Feb 2026 15:53:59 +0000 Subject: [PATCH 07/49] Add dynamic secrets handling for ECS deployments in workflow --- .github/workflows/deploy-reusable.yml | 61 +++++++++++++----- infra/services.json | 89 +++++++++++++++++++++++---- 2 files changed, 121 insertions(+), 29 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index cb8db72..f9bda8f 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -131,6 +131,49 @@ jobs: echo "skip=false" >> $GITHUB_OUTPUT echo "Published image: ${IMAGE}" + - name: Build secrets list for service + id: build-secrets + run: | + # Get the secrets array for this service from services.json + SECRETS_JSON='${{ toJson(matrix.service.secrets) }}' + + # Build the secrets string dynamically + SECRETS_STRING="" + + # Map of secret keys to their connection string names + declare -A SECRET_MAP=( + ["API_Base_URL"]="API__BaseUrl" + ["Authentication_ApiKey"]="Authentication__ApiKey" + ["S3_BucketName"]="AWS__S3__BucketName" + ["Client_ID"]="AzureAd__ClientId" + ["Client_Secret"]="AzureAd__ClientSecret" + ["AuditDb"]="ConnectionStrings__AuditDb" + ["ClusterDb"]="ConnectionStrings__ClusterDb" + ["DeliusRunningPictureDb"]="ConnectionStrings__DeliusRunningPictureDb" + ["DeliusStagingDb"]="ConnectionStrings__DeliusStagingDb" + ["MatchingDb"]="ConnectionStrings__MatchingDb" + ["OfflocRunningPictureDb"]="ConnectionStrings__OfflocRunningPictureDb" + ["OfflocStagingDb"]="ConnectionStrings__OfflocStagingDb" + ["CatsRabbitMQ"]="ConnectionStrings__CatsRabbitMQ" + ["RabbitMQ"]="ConnectionStrings__RabbitMQ" + ["DMSFilesBasePath"]="DMSFilesBasePath" + ["Sentry_Dsn"]="Sentry_Dsn" + ) + + # Parse JSON array and build secrets string + for secret in $(echo "$SECRETS_JSON" | jq -r '.[]'); do + env_var_name="${SECRET_MAP[$secret]}" + if [ -n "$env_var_name" ]; then + SECRETS_STRING+="${env_var_name}=\${{ secrets.AWS_SECRETS_ARN }}:${secret}::"$'\n' + fi + done + + # Remove trailing newline and save + SECRETS_STRING=$(echo "$SECRETS_STRING" | sed '/^$/d') + echo "secrets<> $GITHUB_OUTPUT + echo "$SECRETS_STRING" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Render ECS task definition if: steps.build-image.outputs.skip != 'true' id: render-task-def @@ -143,23 +186,7 @@ jobs: execution-role-arn: ${{ steps.fetch-roles.outputs.exec-role }} environment-variables: | DOTNET_ENVIRONMENT=${{ inputs.environment == 'prod' && 'Production' || 'PreProduction' }} - secrets: | - API__BaseUrl=${{ secrets.AWS_SECRETS_ARN }}:API_Base_URL:: - Authentication__ApiKey=${{ secrets.AWS_SECRETS_ARN }}:Authentication_ApiKey:: - AWS__S3__BucketName=${{ secrets.AWS_SECRETS_ARN }}:S3_BucketName:: - AzureAd__ClientId=${{ secrets.AWS_SECRETS_ARN }}:Client_ID:: - AzureAd__ClientSecret=${{ secrets.AWS_SECRETS_ARN }}:Client_Secret:: - ConnectionStrings__AuditDb=${{ secrets.AWS_SECRETS_ARN }}:AuditDb:: - ConnectionStrings__ClusterDb=${{ secrets.AWS_SECRETS_ARN }}:ClusterDb:: - ConnectionStrings__DeliusRunningPictureDb=${{ secrets.AWS_SECRETS_ARN }}:DeliusRunningPictureDb:: - ConnectionStrings__DeliusStagingDb=${{ secrets.AWS_SECRETS_ARN }}:DeliusStagingDb:: - ConnectionStrings__MatchingDb=${{ secrets.AWS_SECRETS_ARN }}:MatchingDb:: - ConnectionStrings__OfflocRunningPictureDb=${{ secrets.AWS_SECRETS_ARN }}:OfflocRunningPictureDb:: - ConnectionStrings__OfflocStagingDb=${{ secrets.AWS_SECRETS_ARN }}:OfflocStagingDb:: - ConnectionStrings__CatsRabbitMQ=${{ secrets.AWS_SECRETS_ARN }}:CatsRabbitMQ:: - ConnectionStrings__RabbitMQ=${{ secrets.AWS_SECRETS_ARN }}:RabbitMQ:: - DMSFilesBasePath=${{ secrets.AWS_SECRETS_ARN }}:DMSFilesBasePath:: - Sentry_Dsn=${{ secrets.AWS_SECRETS_ARN }}:Sentry_Dsn:: + secrets: ${{ steps.build-secrets.outputs.secrets }} diff --git a/infra/services.json b/infra/services.json index 138c4e9..e275048 100644 --- a/infra/services.json +++ b/infra/services.json @@ -4,73 +4,138 @@ "name": "api", "project": "API/API.csproj", "taskDef": "infra/task-definitions/api-task-def.json", - "container": "api-container" + "container": "api-container", + "secrets": [ + "AuditDb", + "DeliusRunningPictureDb", + "OfflocRunningPictureDb", + "ClusterDb", + "Client_ID", + "Authentication_ApiKey", + "Sentry_Dsn" + ] }, { "name": "offloc-cleaner", "project": "Offloc.Cleaner/Offloc.Cleaner.csproj", "taskDef": "infra/task-definitions/offloc-cleaner-task-def.json", - "container": "offloc-cleaner-container" + "container": "offloc-cleaner-container", + "secrets": [ + "RabbitMQ", + "Sentry_Dsn" + ] }, { "name": "matching-engine", "project": "Matching.Engine/Matching.Engine.csproj", "taskDef": "infra/task-definitions/matching-engine-task-def.json", - "container": "matching-engine-container" + "container": "matching-engine-container", + "secrets": [ + "ClusterDb", + "MatchingDb", + "RabbitMQ", + "Sentry_Dsn" + ] }, { "name": "cleanup", "project": "Cleanup/Cleanup.csproj", "taskDef": "infra/task-definitions/cleanup-task-def.json", - "container": "cleanup-container" + "container": "cleanup-container", + "secrets": [ + "RabbitMQ", + "Sentry_Dsn" + ] }, { "name": "import", "project": "Import/Import.csproj", "taskDef": "infra/task-definitions/import-task-def.json", - "container": "import-container" + "container": "import-container", + "secrets": [ + "RabbitMQ", + "Sentry_Dsn" + ] }, { "name": "visualiser", "project": "Visualiser/Visualiser.csproj", "taskDef": "infra/task-definitions/visualiser-task-def.json", - "container": "visualiser-container" + "container": "visualiser-container", + "secrets": [ + "API_Base_URL", + "Client_ID", + "Client_Secret", + "Sentry_Dsn" + ] }, { "name": "offloc-parser", "project": "Offloc.Parser/Offloc.Parser.csproj", "taskDef": "infra/task-definitions/offloc-parser-task-def.json", - "container": "offloc-parser-container" + "container": "offloc-parser-container", + "secrets": [ + "OfflocStagingDb", + "RabbitMQ", + "Sentry_Dsn" + ] }, { "name": "delius-parser", "project": "Delius.Parser/Delius.Parser.csproj", "taskDef": "infra/task-definitions/delius-parser-task-def.json", - "container": "delius-parser-container" + "container": "delius-parser-container", + "secrets": [ + "DeliusStagingDb", + "RabbitMQ", + "Sentry_Dsn" + ] }, { "name": "blocking", "project": "Blocking/Blocking.csproj", "taskDef": "infra/task-definitions/blocking-task-def.json", - "container": "blocking-container" + "container": "blocking-container", + "secrets": [ + "MatchingDb", + "RabbitMQ", + "Sentry_Dsn" + ] }, { "name": "logging", "project": "Logging/Logging.csproj", "taskDef": "infra/task-definitions/logging-task-def.json", - "container": "logging-container" + "container": "logging-container", + "secrets": [ + "RabbitMQ", + "Sentry_Dsn" + ] }, { "name": "filesync", "project": "FileSync/FileSync.csproj", "taskDef": "infra/task-definitions/filesync-task-def.json", - "container": "filesync-container" + "container": "filesync-container", + "secrets": [ + "S3_BucketName", + "RabbitMQ", + "Sentry_Dsn" + ] }, { "name": "meow", "project": "Meow/Meow.csproj", "taskDef": "infra/task-definitions/meow-task-def.json", - "container": "meow-container" + "container": "meow-container", + "secrets": [ + "AuditDb", + "DeliusRunningPictureDb", + "OfflocRunningPictureDb", + "ClusterDb", + "CatsRabbitMQ", + "Sentry_Dsn" + ] } ] } From 99aa2e004311f962461722df29fefea6c85f04bd Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Wed, 11 Feb 2026 18:55:36 +0000 Subject: [PATCH 08/49] Enhance ECS task definitions with EFS configuration and update IAM role fetching --- .github/workflows/deploy-reusable.yml | 16 +++- infra/task-definitions/blocking-task-def.json | 93 ++++++++++--------- infra/task-definitions/cleanup-task-def.json | 93 ++++++++++--------- .../delius-parser-task-def.json | 93 ++++++++++--------- infra/task-definitions/filesync-task-def.json | 93 ++++++++++--------- infra/task-definitions/import-task-def.json | 93 ++++++++++--------- infra/task-definitions/logging-task-def.json | 93 ++++++++++--------- .../matching-engine-task-def.json | 93 ++++++++++--------- infra/task-definitions/meow-task-def.json | 93 ++++++++++--------- .../offloc-cleaner-task-def.json | 93 ++++++++++--------- .../offloc-parser-task-def.json | 93 ++++++++++--------- 11 files changed, 495 insertions(+), 451 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index f9bda8f..c232f20 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -76,7 +76,7 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - - name: Fetch IAM Role ARNs from Secrets Manager + - name: Fetch IAM Role ARNs and EFS config from Secrets Manager id: fetch-roles run: | SECRET_JSON=$(aws secretsmanager get-secret-value \ @@ -86,13 +86,19 @@ jobs: TASK_ROLE=$(echo "$SECRET_JSON" | jq -r '.Task_Role_ARN') EXEC_ROLE=$(echo "$SECRET_JSON" | jq -r '.Execution_Role_ARN') + EFS_FS_ID=$(echo "$SECRET_JSON" | jq -r '.EFS_File_System_ID') + EFS_AP_ID=$(echo "$SECRET_JSON" | jq -r '.EFS_Access_Point_ID') # Mask secrets in logs echo "::add-mask::$TASK_ROLE" echo "::add-mask::$EXEC_ROLE" + echo "::add-mask::$EFS_FS_ID" + echo "::add-mask::$EFS_AP_ID" echo "task-role=${TASK_ROLE}" >> $GITHUB_OUTPUT echo "exec-role=${EXEC_ROLE}" >> $GITHUB_OUTPUT + echo "efs-fs-id=${EFS_FS_ID}" >> $GITHUB_OUTPUT + echo "efs-ap-id=${EFS_AP_ID}" >> $GITHUB_OUTPUT - name: Setup .NET uses: actions/setup-dotnet@v5 @@ -174,6 +180,14 @@ jobs: echo "$SECRETS_STRING" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT + - name: Replace EFS placeholders in task definition + if: steps.build-image.outputs.skip != 'true' + run: | + sed -i.bak \ + -e 's/EFS_FILE_SYSTEM_ID/${{ steps.fetch-roles.outputs.efs-fs-id }}/g' \ + -e 's/EFS_ACCESS_POINT_ID/${{ steps.fetch-roles.outputs.efs-ap-id }}/g' \ + ${{ matrix.service.taskDef }} + - name: Render ECS task definition if: steps.build-image.outputs.skip != 'true' id: render-task-def diff --git a/infra/task-definitions/blocking-task-def.json b/infra/task-definitions/blocking-task-def.json index 1985ece..259e225 100644 --- a/infra/task-definitions/blocking-task-def.json +++ b/infra/task-definitions/blocking-task-def.json @@ -1,50 +1,53 @@ { - "containerDefinitions": [ + "containerDefinitions": [ + { + "name": "blocking-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ { - "name": "blocking-container", - "image": "PLACEHOLDER", - "cpu": 512, - "memory": 1024, - "essential": true, - "environment": [], - "secrets": [], - "mountPoints": [ - { - "sourceVolume": "dms-efs", - "containerPath": "/mnt/efs", - "readOnly": false - } - ], - "volumesFrom": [], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "/ecs/dms-blocking", - "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" - } - }, - "systemControls": [] + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false } - ], - "family": "dms-blocking-task", - "networkMode": "awsvpc", - "volumes": [ - { - "name": "dms-efs", - "efsVolumeConfiguration": { - "fileSystemId": "EFS_FILE_SYSTEM_ID", - "transitEncryption": "ENABLED" - } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-blocking", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-blocking-task", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED", + "authorizationConfig": { + "accessPointId": "EFS_ACCESS_POINT_ID" } - ], - "placementConstraints": [], - "compatibilities": [ - "FARGATE" - ], - "requiresCompatibilities": [ - "FARGATE" - ], - "cpu": "512", - "memory": "1024" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" } diff --git a/infra/task-definitions/cleanup-task-def.json b/infra/task-definitions/cleanup-task-def.json index 8fe6bc4..813db55 100644 --- a/infra/task-definitions/cleanup-task-def.json +++ b/infra/task-definitions/cleanup-task-def.json @@ -1,50 +1,53 @@ { - "containerDefinitions": [ + "containerDefinitions": [ + { + "name": "cleanup-container", + "image": "PLACEHOLDER", + "cpu": 256, + "memory": 512, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ { - "name": "cleanup-container", - "image": "PLACEHOLDER", - "cpu": 256, - "memory": 512, - "essential": true, - "environment": [], - "secrets": [], - "mountPoints": [ - { - "sourceVolume": "dms-efs", - "containerPath": "/mnt/efs", - "readOnly": false - } - ], - "volumesFrom": [], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "/ecs/dms-cleanup", - "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" - } - }, - "systemControls": [] + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false } - ], - "family": "dms-cleanup-task", - "networkMode": "awsvpc", - "volumes": [ - { - "name": "dms-efs", - "efsVolumeConfiguration": { - "fileSystemId": "EFS_FILE_SYSTEM_ID", - "transitEncryption": "ENABLED" - } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-cleanup", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-cleanup-task", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED", + "authorizationConfig": { + "accessPointId": "EFS_ACCESS_POINT_ID" } - ], - "placementConstraints": [], - "compatibilities": [ - "FARGATE" - ], - "requiresCompatibilities": [ - "FARGATE" - ], - "cpu": "256", - "memory": "512" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "256", + "memory": "512" } diff --git a/infra/task-definitions/delius-parser-task-def.json b/infra/task-definitions/delius-parser-task-def.json index 20b3363..fa000b1 100644 --- a/infra/task-definitions/delius-parser-task-def.json +++ b/infra/task-definitions/delius-parser-task-def.json @@ -1,50 +1,53 @@ { - "containerDefinitions": [ + "containerDefinitions": [ + { + "name": "delius-parser-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ { - "name": "delius-parser-container", - "image": "PLACEHOLDER", - "cpu": 512, - "memory": 1024, - "essential": true, - "environment": [], - "secrets": [], - "mountPoints": [ - { - "sourceVolume": "dms-efs", - "containerPath": "/mnt/efs", - "readOnly": false - } - ], - "volumesFrom": [], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "/ecs/dms-delius-parser", - "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" - } - }, - "systemControls": [] + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false } - ], - "family": "dms-delius-parser-task", - "networkMode": "awsvpc", - "volumes": [ - { - "name": "dms-efs", - "efsVolumeConfiguration": { - "fileSystemId": "EFS_FILE_SYSTEM_ID", - "transitEncryption": "ENABLED" - } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-delius-parser", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-delius-parser-task", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED", + "authorizationConfig": { + "accessPointId": "EFS_ACCESS_POINT_ID" } - ], - "placementConstraints": [], - "compatibilities": [ - "FARGATE" - ], - "requiresCompatibilities": [ - "FARGATE" - ], - "cpu": "512", - "memory": "1024" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" } diff --git a/infra/task-definitions/filesync-task-def.json b/infra/task-definitions/filesync-task-def.json index 6e6629e..1ba6672 100644 --- a/infra/task-definitions/filesync-task-def.json +++ b/infra/task-definitions/filesync-task-def.json @@ -1,50 +1,53 @@ { - "containerDefinitions": [ + "containerDefinitions": [ + { + "name": "filesync-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ { - "name": "filesync-container", - "image": "PLACEHOLDER", - "cpu": 512, - "memory": 1024, - "essential": true, - "environment": [], - "secrets": [], - "mountPoints": [ - { - "sourceVolume": "dms-efs", - "containerPath": "/mnt/efs", - "readOnly": false - } - ], - "volumesFrom": [], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "/ecs/dms-filesync", - "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" - } - }, - "systemControls": [] + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false } - ], - "family": "dms-filesync-task", - "networkMode": "awsvpc", - "volumes": [ - { - "name": "dms-efs", - "efsVolumeConfiguration": { - "fileSystemId": "EFS_FILE_SYSTEM_ID", - "transitEncryption": "ENABLED" - } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-filesync", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-filesync-task", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED", + "authorizationConfig": { + "accessPointId": "EFS_ACCESS_POINT_ID" } - ], - "placementConstraints": [], - "compatibilities": [ - "FARGATE" - ], - "requiresCompatibilities": [ - "FARGATE" - ], - "cpu": "512", - "memory": "1024" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" } diff --git a/infra/task-definitions/import-task-def.json b/infra/task-definitions/import-task-def.json index 2e253c1..6b1c827 100644 --- a/infra/task-definitions/import-task-def.json +++ b/infra/task-definitions/import-task-def.json @@ -1,50 +1,53 @@ { - "containerDefinitions": [ + "containerDefinitions": [ + { + "name": "import-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ { - "name": "import-container", - "image": "PLACEHOLDER", - "cpu": 512, - "memory": 1024, - "essential": true, - "environment": [], - "secrets": [], - "mountPoints": [ - { - "sourceVolume": "dms-efs", - "containerPath": "/mnt/efs", - "readOnly": false - } - ], - "volumesFrom": [], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "/ecs/dms-import", - "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" - } - }, - "systemControls": [] + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false } - ], - "family": "dms-import-task", - "networkMode": "awsvpc", - "volumes": [ - { - "name": "dms-efs", - "efsVolumeConfiguration": { - "fileSystemId": "EFS_FILE_SYSTEM_ID", - "transitEncryption": "ENABLED" - } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-import", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-import-task", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED", + "authorizationConfig": { + "accessPointId": "EFS_ACCESS_POINT_ID" } - ], - "placementConstraints": [], - "compatibilities": [ - "FARGATE" - ], - "requiresCompatibilities": [ - "FARGATE" - ], - "cpu": "512", - "memory": "1024" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" } diff --git a/infra/task-definitions/logging-task-def.json b/infra/task-definitions/logging-task-def.json index 984471d..f977aa1 100644 --- a/infra/task-definitions/logging-task-def.json +++ b/infra/task-definitions/logging-task-def.json @@ -1,50 +1,53 @@ { - "containerDefinitions": [ + "containerDefinitions": [ + { + "name": "logging-container", + "image": "PLACEHOLDER", + "cpu": 256, + "memory": 512, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ { - "name": "logging-container", - "image": "PLACEHOLDER", - "cpu": 256, - "memory": 512, - "essential": true, - "environment": [], - "secrets": [], - "mountPoints": [ - { - "sourceVolume": "dms-efs", - "containerPath": "/mnt/efs", - "readOnly": false - } - ], - "volumesFrom": [], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "/ecs/dms-logging", - "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" - } - }, - "systemControls": [] + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false } - ], - "family": "dms-logging-task", - "networkMode": "awsvpc", - "volumes": [ - { - "name": "dms-efs", - "efsVolumeConfiguration": { - "fileSystemId": "EFS_FILE_SYSTEM_ID", - "transitEncryption": "ENABLED" - } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-logging", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-logging-task", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED", + "authorizationConfig": { + "accessPointId": "EFS_ACCESS_POINT_ID" } - ], - "placementConstraints": [], - "compatibilities": [ - "FARGATE" - ], - "requiresCompatibilities": [ - "FARGATE" - ], - "cpu": "256", - "memory": "512" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "256", + "memory": "512" } diff --git a/infra/task-definitions/matching-engine-task-def.json b/infra/task-definitions/matching-engine-task-def.json index 5b3398d..e7fe8d7 100644 --- a/infra/task-definitions/matching-engine-task-def.json +++ b/infra/task-definitions/matching-engine-task-def.json @@ -1,50 +1,53 @@ { - "containerDefinitions": [ + "containerDefinitions": [ + { + "name": "matching-engine-container", + "image": "PLACEHOLDER", + "cpu": 1024, + "memory": 2048, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ { - "name": "matching-engine-container", - "image": "PLACEHOLDER", - "cpu": 1024, - "memory": 2048, - "essential": true, - "environment": [], - "secrets": [], - "mountPoints": [ - { - "sourceVolume": "dms-efs", - "containerPath": "/mnt/efs", - "readOnly": false - } - ], - "volumesFrom": [], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "/ecs/dms-matching-engine", - "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" - } - }, - "systemControls": [] + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false } - ], - "family": "dms-matching-engine-task", - "networkMode": "awsvpc", - "volumes": [ - { - "name": "dms-efs", - "efsVolumeConfiguration": { - "fileSystemId": "EFS_FILE_SYSTEM_ID", - "transitEncryption": "ENABLED" - } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-matching-engine", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-matching-engine-task", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED", + "authorizationConfig": { + "accessPointId": "EFS_ACCESS_POINT_ID" } - ], - "placementConstraints": [], - "compatibilities": [ - "FARGATE" - ], - "requiresCompatibilities": [ - "FARGATE" - ], - "cpu": "1024", - "memory": "2048" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "1024", + "memory": "2048" } diff --git a/infra/task-definitions/meow-task-def.json b/infra/task-definitions/meow-task-def.json index f2d7319..50cd3a3 100644 --- a/infra/task-definitions/meow-task-def.json +++ b/infra/task-definitions/meow-task-def.json @@ -1,50 +1,53 @@ { - "containerDefinitions": [ + "containerDefinitions": [ + { + "name": "meow-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ { - "name": "meow-container", - "image": "PLACEHOLDER", - "cpu": 512, - "memory": 1024, - "essential": true, - "environment": [], - "secrets": [], - "mountPoints": [ - { - "sourceVolume": "dms-efs", - "containerPath": "/mnt/efs", - "readOnly": false - } - ], - "volumesFrom": [], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "/ecs/dms-meow", - "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" - } - }, - "systemControls": [] + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false } - ], - "family": "dms-meow-task", - "networkMode": "awsvpc", - "volumes": [ - { - "name": "dms-efs", - "efsVolumeConfiguration": { - "fileSystemId": "EFS_FILE_SYSTEM_ID", - "transitEncryption": "ENABLED" - } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-meow", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-meow-task", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED", + "authorizationConfig": { + "accessPointId": "EFS_ACCESS_POINT_ID" } - ], - "placementConstraints": [], - "compatibilities": [ - "FARGATE" - ], - "requiresCompatibilities": [ - "FARGATE" - ], - "cpu": "512", - "memory": "1024" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" } diff --git a/infra/task-definitions/offloc-cleaner-task-def.json b/infra/task-definitions/offloc-cleaner-task-def.json index 2811bb8..75b81b3 100644 --- a/infra/task-definitions/offloc-cleaner-task-def.json +++ b/infra/task-definitions/offloc-cleaner-task-def.json @@ -1,50 +1,53 @@ { - "containerDefinitions": [ + "containerDefinitions": [ + { + "name": "offloc-cleaner-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ { - "name": "offloc-cleaner-container", - "image": "PLACEHOLDER", - "cpu": 512, - "memory": 1024, - "essential": true, - "environment": [], - "secrets": [], - "mountPoints": [ - { - "sourceVolume": "dms-efs", - "containerPath": "/mnt/efs", - "readOnly": false - } - ], - "volumesFrom": [], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "/ecs/dms-offloc-cleaner", - "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" - } - }, - "systemControls": [] + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false } - ], - "family": "dms-offloc-cleaner-task", - "networkMode": "awsvpc", - "volumes": [ - { - "name": "dms-efs", - "efsVolumeConfiguration": { - "fileSystemId": "EFS_FILE_SYSTEM_ID", - "transitEncryption": "ENABLED" - } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-offloc-cleaner", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-offloc-cleaner-task", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED", + "authorizationConfig": { + "accessPointId": "EFS_ACCESS_POINT_ID" } - ], - "placementConstraints": [], - "compatibilities": [ - "FARGATE" - ], - "requiresCompatibilities": [ - "FARGATE" - ], - "cpu": "512", - "memory": "1024" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" } diff --git a/infra/task-definitions/offloc-parser-task-def.json b/infra/task-definitions/offloc-parser-task-def.json index faa3b5e..a8aa686 100644 --- a/infra/task-definitions/offloc-parser-task-def.json +++ b/infra/task-definitions/offloc-parser-task-def.json @@ -1,50 +1,53 @@ { - "containerDefinitions": [ + "containerDefinitions": [ + { + "name": "offloc-parser-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ { - "name": "offloc-parser-container", - "image": "PLACEHOLDER", - "cpu": 512, - "memory": 1024, - "essential": true, - "environment": [], - "secrets": [], - "mountPoints": [ - { - "sourceVolume": "dms-efs", - "containerPath": "/mnt/efs", - "readOnly": false - } - ], - "volumesFrom": [], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "/ecs/dms-offloc-parser", - "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" - } - }, - "systemControls": [] + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false } - ], - "family": "dms-offloc-parser-task", - "networkMode": "awsvpc", - "volumes": [ - { - "name": "dms-efs", - "efsVolumeConfiguration": { - "fileSystemId": "EFS_FILE_SYSTEM_ID", - "transitEncryption": "ENABLED" - } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-offloc-parser", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-offloc-parser-task", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED", + "authorizationConfig": { + "accessPointId": "EFS_ACCESS_POINT_ID" } - ], - "placementConstraints": [], - "compatibilities": [ - "FARGATE" - ], - "requiresCompatibilities": [ - "FARGATE" - ], - "cpu": "512", - "memory": "1024" + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" } From 1a89d3b1bdc9f4695e6cb677046b59c30bb8bce6 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Thu, 12 Feb 2026 14:23:36 +0000 Subject: [PATCH 09/49] Add dbinteractions service and task definition for ECS deployment --- infra/services.json | 14 +++++ .../dbinteractions-task-def.json | 53 +++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 infra/task-definitions/dbinteractions-task-def.json diff --git a/infra/services.json b/infra/services.json index e275048..2fc6e95 100644 --- a/infra/services.json +++ b/infra/services.json @@ -91,6 +91,20 @@ "Sentry_Dsn" ] }, + { + "name": "dbinteractions", + "project": "DbInteractions/DbInteractions.csproj", + "taskDef": "infra/task-definitions/dbinteractions-task-def.json", + "container": "dbinteractions-container", + "secrets": [ + "DeliusRunningPictureDb", + "OfflocRunningPictureDb", + "DeliusStagingDb", + "OfflocStagingDb", + "RabbitMQ", + "Sentry_Dsn" + ] + }, { "name": "blocking", "project": "Blocking/Blocking.csproj", diff --git a/infra/task-definitions/dbinteractions-task-def.json b/infra/task-definitions/dbinteractions-task-def.json new file mode 100644 index 0000000..b6d0d2a --- /dev/null +++ b/infra/task-definitions/dbinteractions-task-def.json @@ -0,0 +1,53 @@ +{ + "containerDefinitions": [ + { + "name": "dbinteractions-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [ + { + "sourceVolume": "dms-efs", + "containerPath": "/mnt/efs", + "readOnly": false + } + ], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-dbinteractions", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" + } + }, + "systemControls": [] + } + ], + "family": "dms-dbinteractions-task", + "networkMode": "awsvpc", + "volumes": [ + { + "name": "dms-efs", + "efsVolumeConfiguration": { + "fileSystemId": "EFS_FILE_SYSTEM_ID", + "transitEncryption": "ENABLED", + "authorizationConfig": { + "accessPointId": "EFS_ACCESS_POINT_ID" + } + } + } + ], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" +} From 7c9f60be8b36c859b37a646350d0dff7adc6e780 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Thu, 12 Feb 2026 14:32:32 +0000 Subject: [PATCH 10/49] Update ECS service and cluster naming conventions based on environment --- .github/workflows/deploy-reusable.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index c232f20..8d95c2c 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -209,6 +209,6 @@ jobs: uses: aws-actions/amazon-ecs-deploy-task-definition@v2 with: task-definition: ${{ steps.render-task-def.outputs.task-definition }} - service: ${{ matrix.service.name }}-service-1 - cluster: ${{ matrix.service.name }}-cluster + service: DMS-${{ inputs.environment == 'prod' && 'Prod' || 'PreProd' }}-${{ matrix.service.name }}-service + cluster: DMS-${{ inputs.environment == 'prod' && 'Prod' || 'PreProd' }}-Cluster wait-for-service-stability: true From a889a8c14fb98acd7c1dfc2c91259acc64a9aa9a Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Thu, 19 Feb 2026 11:47:44 +0000 Subject: [PATCH 11/49] Add DMSFilesBasePath secret to multiple services in services.json --- infra/services.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/infra/services.json b/infra/services.json index 2fc6e95..557bc39 100644 --- a/infra/services.json +++ b/infra/services.json @@ -21,6 +21,7 @@ "taskDef": "infra/task-definitions/offloc-cleaner-task-def.json", "container": "offloc-cleaner-container", "secrets": [ + "DMSFilesBasePath", "RabbitMQ", "Sentry_Dsn" ] @@ -43,6 +44,7 @@ "taskDef": "infra/task-definitions/cleanup-task-def.json", "container": "cleanup-container", "secrets": [ + "DMSFilesBasePath", "RabbitMQ", "Sentry_Dsn" ] @@ -75,6 +77,7 @@ "taskDef": "infra/task-definitions/offloc-parser-task-def.json", "container": "offloc-parser-container", "secrets": [ + "DMSFilesBasePath", "OfflocStagingDb", "RabbitMQ", "Sentry_Dsn" @@ -86,6 +89,7 @@ "taskDef": "infra/task-definitions/delius-parser-task-def.json", "container": "delius-parser-container", "secrets": [ + "DMSFilesBasePath", "DeliusStagingDb", "RabbitMQ", "Sentry_Dsn" @@ -97,6 +101,7 @@ "taskDef": "infra/task-definitions/dbinteractions-task-def.json", "container": "dbinteractions-container", "secrets": [ + "DMSFilesBasePath", "DeliusRunningPictureDb", "OfflocRunningPictureDb", "DeliusStagingDb", @@ -132,6 +137,7 @@ "taskDef": "infra/task-definitions/filesync-task-def.json", "container": "filesync-container", "secrets": [ + "DMSFilesBasePath", "S3_BucketName", "RabbitMQ", "Sentry_Dsn" From 4ece70af1b9bc7ec82db5a2beaf7225bdaf12b9b Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Fri, 20 Feb 2026 09:13:02 +0000 Subject: [PATCH 12/49] Update image repository structure and tagging format in deploy workflow --- .github/workflows/deploy-reusable.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 8d95c2c..bd79833 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -112,8 +112,8 @@ jobs: id: build-image env: REGISTRY: ${{ steps.login-ecr.outputs.registry }} - REPOSITORY: ${{ inputs.environment }}/${{ matrix.service.name }} - IMAGE_TAG: ${{ github.sha }} + REPOSITORY: dms-${{ inputs.environment }}-apps + IMAGE_TAG: ${{ matrix.service.name }}-${{ github.sha }} run: | PROJECT_PATH="src/${{ matrix.service.project }}" From 59260e997e6786c94f8f7d4138b54449a494dcf3 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Fri, 20 Feb 2026 09:36:04 +0000 Subject: [PATCH 13/49] Add debug logging for deployment information in reusable deploy workflow --- .github/workflows/deploy-reusable.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index bd79833..7855459 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -66,6 +66,15 @@ jobs: - name: Checkout uses: actions/checkout@v5 + # Debug logging + - name: Print debug info + run: | + echo "Deploying to environment: ${{ inputs.environment }}" + echo "Deploying service: ${{ matrix.service.name }}" + echo "Project path: src/${{ matrix.service.project }}" + echo "Ref: ${{ github.ref }}" + echo "Repository: ${{ github.repository }}" + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: From d183a0cf3881c0e0651f14700a648e1897879136 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Fri, 20 Feb 2026 13:31:03 +0000 Subject: [PATCH 14/49] Add debug logging to confirm AWS credentials in deploy workflow --- .github/workflows/deploy-reusable.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 7855459..2fc537a 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -81,6 +81,10 @@ jobs: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} aws-region: ${{ inputs.aws-region }} + # Debug logging + - name: Confirm AWS credentials + run: aws sts get-caller-identity + - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v2 From e4d77398dd74e44393f8109ff7887ffd4f326701 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Fri, 20 Feb 2026 13:59:16 +0000 Subject: [PATCH 15/49] Remove obsolete Aspire.Hosting.AppHost package version from Directory.Packages.props - causing conflict in workflow --- Directory.Packages.props | 1 - 1 file changed, 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index d883f0c..d299cb9 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,7 +3,6 @@ true - From bf1380dcfe073d646b607e462915ed2d48cd3132 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 10:12:59 +0000 Subject: [PATCH 16/49] Inject IAM roles into ECS task definition in deploy workflow --- .github/workflows/deploy-reusable.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 2fc537a..c57724e 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -201,6 +201,15 @@ jobs: -e 's/EFS_ACCESS_POINT_ID/${{ steps.fetch-roles.outputs.efs-ap-id }}/g' \ ${{ matrix.service.taskDef }} + - name: Inject IAM roles into task definition + if: steps.build-image.outputs.skip != 'true' + run: | + jq --arg taskRole "${{ steps.fetch-roles.outputs.task-role }}" \ + --arg execRole "${{ steps.fetch-roles.outputs.exec-role }}" \ + '.taskRoleArn = $taskRole | .executionRoleArn = $execRole' \ + ${{ matrix.service.taskDef }} > task-def-updated.json + mv task-def-updated.json ${{ matrix.service.taskDef }} + - name: Render ECS task definition if: steps.build-image.outputs.skip != 'true' id: render-task-def @@ -209,8 +218,6 @@ jobs: task-definition: ${{ matrix.service.taskDef }} container-name: ${{ matrix.service.container }} image: ${{ steps.build-image.outputs.image }} - task-role-arn: ${{ steps.fetch-roles.outputs.task-role }} - execution-role-arn: ${{ steps.fetch-roles.outputs.exec-role }} environment-variables: | DOTNET_ENVIRONMENT=${{ inputs.environment == 'prod' && 'Production' || 'PreProduction' }} secrets: ${{ steps.build-secrets.outputs.secrets }} From 3d1365fd51338b6ff935d822ffb4caf4d78eccd9 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 10:50:58 +0000 Subject: [PATCH 17/49] Testing: Working through wrokflow to ensure steps are correct. First checking if secrets are retrieved correctly --- .github/workflows/deploy-reusable.yml | 184 ++++++-------------------- 1 file changed, 41 insertions(+), 143 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index c57724e..2f7bbd4 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -66,169 +66,67 @@ jobs: - name: Checkout uses: actions/checkout@v5 - # Debug logging - - name: Print debug info - run: | - echo "Deploying to environment: ${{ inputs.environment }}" - echo "Deploying service: ${{ matrix.service.name }}" - echo "Project path: src/${{ matrix.service.project }}" - echo "Ref: ${{ github.ref }}" - echo "Repository: ${{ github.repository }}" - - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} aws-region: ${{ inputs.aws-region }} - # Debug logging - - name: Confirm AWS credentials - run: aws sts get-caller-identity - - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - - name: Fetch IAM Role ARNs and EFS config from Secrets Manager - id: fetch-roles - run: | - SECRET_JSON=$(aws secretsmanager get-secret-value \ - --secret-id ${{ secrets.AWS_SECRETS_ARN }} \ - --query 'SecretString' \ - --output text) - - TASK_ROLE=$(echo "$SECRET_JSON" | jq -r '.Task_Role_ARN') - EXEC_ROLE=$(echo "$SECRET_JSON" | jq -r '.Execution_Role_ARN') - EFS_FS_ID=$(echo "$SECRET_JSON" | jq -r '.EFS_File_System_ID') - EFS_AP_ID=$(echo "$SECRET_JSON" | jq -r '.EFS_Access_Point_ID') - - # Mask secrets in logs - echo "::add-mask::$TASK_ROLE" - echo "::add-mask::$EXEC_ROLE" - echo "::add-mask::$EFS_FS_ID" - echo "::add-mask::$EFS_AP_ID" - - echo "task-role=${TASK_ROLE}" >> $GITHUB_OUTPUT - echo "exec-role=${EXEC_ROLE}" >> $GITHUB_OUTPUT - echo "efs-fs-id=${EFS_FS_ID}" >> $GITHUB_OUTPUT - echo "efs-ap-id=${EFS_AP_ID}" >> $GITHUB_OUTPUT - - name: Setup .NET uses: actions/setup-dotnet@v5 with: dotnet-version: 10.0.x - - name: Run tests - run: dotnet test --configuration Release - - - name: Build and Publish Image - id: build-image - env: - REGISTRY: ${{ steps.login-ecr.outputs.registry }} - REPOSITORY: dms-${{ inputs.environment }}-apps - IMAGE_TAG: ${{ matrix.service.name }}-${{ github.sha }} - run: | - PROJECT_PATH="src/${{ matrix.service.project }}" + # - name: Run tests + # run: dotnet test --configuration Release + + # - name: Build and Publish Image + # id: build-image + # env: + # REGISTRY: ${{ steps.login-ecr.outputs.registry }} + # REPOSITORY: dms-${{ inputs.environment }}-apps + # IMAGE_TAG: ${{ matrix.service.name }}-${{ github.sha }} + # run: | + # PROJECT_PATH="src/${{ matrix.service.project }}" - if [ ! -f "$PROJECT_PATH" ]; then - echo "Project not found: $PROJECT_PATH - Skipping" - echo "skip=true" >> $GITHUB_OUTPUT - exit 0 - fi + # if [ ! -f "$PROJECT_PATH" ]; then + # echo "Project not found: $PROJECT_PATH - Skipping" + # echo "skip=true" >> $GITHUB_OUTPUT + # exit 0 + # fi - echo "Building and publishing: $PROJECT_PATH" + # echo "Building and publishing: $PROJECT_PATH" - dotnet publish "$PROJECT_PATH" \ - --configuration Release \ - --target:PublishContainer \ - --property:ContainerRegistry=$REGISTRY \ - --property:ContainerRepository=$REPOSITORY \ - --property:ContainerImageTag=$IMAGE_TAG + # dotnet publish "$PROJECT_PATH" \ + # --configuration Release \ + # --target:PublishContainer \ + # --property:ContainerRegistry=$REGISTRY \ + # --property:ContainerRepository=$REPOSITORY \ + # --property:ContainerImageTag=$IMAGE_TAG - IMAGE="${REGISTRY}/${REPOSITORY}:${IMAGE_TAG}" - echo "IMAGE=${IMAGE}" >> $GITHUB_OUTPUT - echo "skip=false" >> $GITHUB_OUTPUT - echo "Published image: ${IMAGE}" + # IMAGE="${REGISTRY}/${REPOSITORY}:${IMAGE_TAG}" + # echo "IMAGE=${IMAGE}" >> $GITHUB_OUTPUT + # echo "skip=false" >> $GITHUB_OUTPUT + # echo "Published image: ${IMAGE}" - - name: Build secrets list for service - id: build-secrets - run: | - # Get the secrets array for this service from services.json - SECRETS_JSON='${{ toJson(matrix.service.secrets) }}' - - # Build the secrets string dynamically - SECRETS_STRING="" - - # Map of secret keys to their connection string names - declare -A SECRET_MAP=( - ["API_Base_URL"]="API__BaseUrl" - ["Authentication_ApiKey"]="Authentication__ApiKey" - ["S3_BucketName"]="AWS__S3__BucketName" - ["Client_ID"]="AzureAd__ClientId" - ["Client_Secret"]="AzureAd__ClientSecret" - ["AuditDb"]="ConnectionStrings__AuditDb" - ["ClusterDb"]="ConnectionStrings__ClusterDb" - ["DeliusRunningPictureDb"]="ConnectionStrings__DeliusRunningPictureDb" - ["DeliusStagingDb"]="ConnectionStrings__DeliusStagingDb" - ["MatchingDb"]="ConnectionStrings__MatchingDb" - ["OfflocRunningPictureDb"]="ConnectionStrings__OfflocRunningPictureDb" - ["OfflocStagingDb"]="ConnectionStrings__OfflocStagingDb" - ["CatsRabbitMQ"]="ConnectionStrings__CatsRabbitMQ" - ["RabbitMQ"]="ConnectionStrings__RabbitMQ" - ["DMSFilesBasePath"]="DMSFilesBasePath" - ["Sentry_Dsn"]="Sentry_Dsn" - ) - - # Parse JSON array and build secrets string - for secret in $(echo "$SECRETS_JSON" | jq -r '.[]'); do - env_var_name="${SECRET_MAP[$secret]}" - if [ -n "$env_var_name" ]; then - SECRETS_STRING+="${env_var_name}=\${{ secrets.AWS_SECRETS_ARN }}:${secret}::"$'\n' - fi - done - - # Remove trailing newline and save - SECRETS_STRING=$(echo "$SECRETS_STRING" | sed '/^$/d') - echo "secrets<> $GITHUB_OUTPUT - echo "$SECRETS_STRING" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - - name: Replace EFS placeholders in task definition - if: steps.build-image.outputs.skip != 'true' - run: | - sed -i.bak \ - -e 's/EFS_FILE_SYSTEM_ID/${{ steps.fetch-roles.outputs.efs-fs-id }}/g' \ - -e 's/EFS_ACCESS_POINT_ID/${{ steps.fetch-roles.outputs.efs-ap-id }}/g' \ - ${{ matrix.service.taskDef }} - - - name: Inject IAM roles into task definition - if: steps.build-image.outputs.skip != 'true' + - name: Testing workflow steps + run: echo "The testing / deployment of the image have been skipped for this demo, but the workflow is correctly iterating over the services and would deploy the built image to ECS if those steps were uncommented." + + - name : Fetch required secrets + id: fetch-secrets run: | - jq --arg taskRole "${{ steps.fetch-roles.outputs.task-role }}" \ - --arg execRole "${{ steps.fetch-roles.outputs.exec-role }}" \ - '.taskRoleArn = $taskRole | .executionRoleArn = $execRole' \ - ${{ matrix.service.taskDef }} > task-def-updated.json - mv task-def-updated.json ${{ matrix.service.taskDef }} - - - name: Render ECS task definition - if: steps.build-image.outputs.skip != 'true' - id: render-task-def - uses: aws-actions/amazon-ecs-render-task-definition@v1.8.1 - with: - task-definition: ${{ matrix.service.taskDef }} - container-name: ${{ matrix.service.container }} - image: ${{ steps.build-image.outputs.image }} - environment-variables: | - DOTNET_ENVIRONMENT=${{ inputs.environment == 'prod' && 'Production' || 'PreProduction' }} - secrets: ${{ steps.build-secrets.outputs.secrets }} - + SECRET_JSON=$(aws secretsmanager get-secret-value \ + --secret-id ${{ secrets.AWS_SECRETS_ARN }} \ + --query 'SecretString' \ + --output text) + TASK_ROLE=$(echo "$SECRET_JSON" | jq -r '.Task_Role_ARN') + EXEC_ROLE=$(echo "$SECRET_JSON" | jq -r '.Execution_Role_ARN') - - name: Deploy ECS service - if: steps.build-image.outputs.skip != 'true' - uses: aws-actions/amazon-ecs-deploy-task-definition@v2 - with: - task-definition: ${{ steps.render-task-def.outputs.task-definition }} - service: DMS-${{ inputs.environment == 'prod' && 'Prod' || 'PreProd' }}-${{ matrix.service.name }}-service - cluster: DMS-${{ inputs.environment == 'prod' && 'Prod' || 'PreProd' }}-Cluster - wait-for-service-stability: true + echo "Task/Exec role arns (not masked as they are not sensitive):" + echo "TASK_ROLE=${TASK_ROLE}" >> $GITHUB_OUTPUT + echo "EXEC_ROLE=${EXEC_ROLE}" >> $GITHUB_OUTPUT \ No newline at end of file From 4525c04a5fa067f5e632a4e2015b3691423a57d7 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 10:58:16 +0000 Subject: [PATCH 18/49] Testing secret retrieval --- .github/workflows/deploy-reusable.yml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 2f7bbd4..5acf19d 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -116,17 +116,20 @@ jobs: - name: Testing workflow steps run: echo "The testing / deployment of the image have been skipped for this demo, but the workflow is correctly iterating over the services and would deploy the built image to ECS if those steps were uncommented." - - name : Fetch required secrets + - name: Fetch required secrets id: fetch-secrets run: | SECRET_JSON=$(aws secretsmanager get-secret-value \ - --secret-id ${{ secrets.AWS_SECRETS_ARN }} \ - --query 'SecretString' \ - --output text) - - TASK_ROLE=$(echo "$SECRET_JSON" | jq -r '.Task_Role_ARN') - EXEC_ROLE=$(echo "$SECRET_JSON" | jq -r '.Execution_Role_ARN') + --secret-id ${{ secrets.AWS_SECRETS_ARN }} \ + --query 'SecretString' \ + --output text) - echo "Task/Exec role arns (not masked as they are not sensitive):" - echo "TASK_ROLE=${TASK_ROLE}" >> $GITHUB_OUTPUT - echo "EXEC_ROLE=${EXEC_ROLE}" >> $GITHUB_OUTPUT \ No newline at end of file + TASK_ROLE=$(echo "$SECRET_JSON" | jq -r '.Task_Role_ARN') + EXEC_ROLE=$(echo "$SECRET_JSON" | jq -r '.Execution_Role_ARN') + + echo "Task/Exec role ARNs:" + echo "Task Role: $TASK_ROLE" + echo "Exec Role: $EXEC_ROLE" + + echo "task_role=$TASK_ROLE" >> $GITHUB_OUTPUT + echo "exec_role=$EXEC_ROLE" >> $GITHUB_OUTPUT \ No newline at end of file From 1c9ebe2fba965ef28417dbe1ec156d3691c9c125 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 11:27:26 +0000 Subject: [PATCH 19/49] Testing: Add ECS secrets building step in deploy workflow --- .github/workflows/deploy-reusable.yml | 50 ++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 5acf19d..7a9588b 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -127,9 +127,49 @@ jobs: TASK_ROLE=$(echo "$SECRET_JSON" | jq -r '.Task_Role_ARN') EXEC_ROLE=$(echo "$SECRET_JSON" | jq -r '.Execution_Role_ARN') - echo "Task/Exec role ARNs:" - echo "Task Role: $TASK_ROLE" - echo "Exec Role: $EXEC_ROLE" - echo "task_role=$TASK_ROLE" >> $GITHUB_OUTPUT - echo "exec_role=$EXEC_ROLE" >> $GITHUB_OUTPUT \ No newline at end of file + echo "exec_role=$EXEC_ROLE" >> $GITHUB_OUTPUT + + - name: Build ECS secrets for service + id: build-secrets + run: | + SECRETS_JSON="${{ toJson(matrix.service.secrets) }}" + SERVICE_NAME="${{ matrix.service.name }}" + SECRET_ARN="${{ secrets.AWS_SECRETS_ARN }}" + + declare -A SECRET_MAP=( + ["API_Base_URL"]="API__BaseUrl" + ["Authentication_ApiKey"]="Authentication__ApiKey" + ["S3_BucketName"]="AWS__S3__BucketName" + ["Client_ID"]="AzureAd__ClientId" + ["Client_Secret"]="AzureAd__ClientSecret" + ["AuditDb"]="ConnectionStrings__AuditDb" + ["ClusterDb"]="ConnectionStrings__ClusterDb" + ["DeliusRunningPictureDb"]="ConnectionStrings__DeliusRunningPictureDb" + ["DeliusStagingDb"]="ConnectionStrings__DeliusStagingDb" + ["MatchingDb"]="ConnectionStrings__MatchingDb" + ["OfflocRunningPictureDb"]="ConnectionStrings__OfflocRunningPictureDb" + ["OfflocStagingDb"]="ConnectionStrings__OfflocStagingDb" + ["CatsRabbitMQ"]="ConnectionStrings__CatsRabbitMQ" + ["RabbitMQ"]="ConnectionStrings__RabbitMQ" + ["DMSFilesBasePath"]="DMSFilesBasePath" + ["Sentry_Dsn"]="Sentry_Dsn" + ) + + SECRETS_LIST="" + for secret in $(echo "$SECRETS_JSON" | jq -r '.[]'); do + ENV_NAME="${SECRET_MAP[$secret]}" + if [ -n "$ENV_NAME" ]; then + SECRETS_LIST+="${ENV_NAME}=${SECRET_ARN}:${secret}::"$'\n' + fi + done + + SECRETS_LIST=$(echo "$SECRETS_LIST" | sed '/^$/d') + + echo "### DEBUG ECS SECRETS - SERVICE: $SERVICE_NAME ###" + echo "$SECRETS_LIST" + echo "### END DEBUG ###" + + echo "secrets<> $GITHUB_OUTPUT + echo "$SECRETS_LIST" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT \ No newline at end of file From 62b8244cb9357c24f4752fe5a751693a8e2fd073 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 11:36:12 +0000 Subject: [PATCH 20/49] Testing: Removing quotes to correctly parse services json --- .github/workflows/deploy-reusable.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 7a9588b..d7b588c 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -133,9 +133,9 @@ jobs: - name: Build ECS secrets for service id: build-secrets run: | - SECRETS_JSON="${{ toJson(matrix.service.secrets) }}" - SERVICE_NAME="${{ matrix.service.name }}" - SECRET_ARN="${{ secrets.AWS_SECRETS_ARN }}" + SECRETS_JSON=${{ toJson(matrix.service.secrets) }} + SERVICE_NAME=${{ matrix.service.name }} + SECRET_ARN=${{ secrets.AWS_SECRETS_ARN }} declare -A SECRET_MAP=( ["API_Base_URL"]="API__BaseUrl" From 2ac10c7da737483d033a86d8c51a76a8ba28d10e Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 11:42:02 +0000 Subject: [PATCH 21/49] Testing: Fix secret retrieval loop to use while read for better parsing --- .github/workflows/deploy-reusable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index d7b588c..1f094ba 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -157,7 +157,7 @@ jobs: ) SECRETS_LIST="" - for secret in $(echo "$SECRETS_JSON" | jq -r '.[]'); do + echo "$SECRETS_JSON" | jq -r '.[]' | while read -r secret; do ENV_NAME="${SECRET_MAP[$secret]}" if [ -n "$ENV_NAME" ]; then SECRETS_LIST+="${ENV_NAME}=${SECRET_ARN}:${secret}::"$'\n' From cd859fc8f357547dfe2e356fcedb13d158b8ca64 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 11:44:52 +0000 Subject: [PATCH 22/49] Testing: Refactor ECS secrets building step for improved string handling --- .github/workflows/deploy-reusable.yml | 33 ++++++++++++--------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 1f094ba..9be2eab 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -130,13 +130,13 @@ jobs: echo "task_role=$TASK_ROLE" >> $GITHUB_OUTPUT echo "exec_role=$EXEC_ROLE" >> $GITHUB_OUTPUT - - name: Build ECS secrets for service + - name: Build secrets list for service id: build-secrets run: | - SECRETS_JSON=${{ toJson(matrix.service.secrets) }} - SERVICE_NAME=${{ matrix.service.name }} - SECRET_ARN=${{ secrets.AWS_SECRETS_ARN }} - + SECRETS_JSON='${{ toJson(matrix.service.secrets) }}' + + SECRETS_STRING="" + declare -A SECRET_MAP=( ["API_Base_URL"]="API__BaseUrl" ["Authentication_ApiKey"]="Authentication__ApiKey" @@ -155,21 +155,18 @@ jobs: ["DMSFilesBasePath"]="DMSFilesBasePath" ["Sentry_Dsn"]="Sentry_Dsn" ) - - SECRETS_LIST="" - echo "$SECRETS_JSON" | jq -r '.[]' | while read -r secret; do - ENV_NAME="${SECRET_MAP[$secret]}" - if [ -n "$ENV_NAME" ]; then - SECRETS_LIST+="${ENV_NAME}=${SECRET_ARN}:${secret}::"$'\n' + + for secret in $(echo "$SECRETS_JSON" | jq -r '.[]'); do + env_var_name="${SECRET_MAP[$secret]}" + if [ -n "$env_var_name" ]; then + SECRETS_STRING+="${env_var_name}=\${{ secrets.AWS_SECRETS_ARN }}:${secret}::"$'\n' fi done - SECRETS_LIST=$(echo "$SECRETS_LIST" | sed '/^$/d') - - echo "### DEBUG ECS SECRETS - SERVICE: $SERVICE_NAME ###" - echo "$SECRETS_LIST" - echo "### END DEBUG ###" - + echo "Built secrets string:" + echo "$SECRETS_STRING" + + SECRETS_STRING=$(echo "$SECRETS_STRING" | sed '/^$/d') echo "secrets<> $GITHUB_OUTPUT - echo "$SECRETS_LIST" >> $GITHUB_OUTPUT + echo "$SECRETS_STRING" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT \ No newline at end of file From 11f634363c5a9dfbe684eb1eec26c6e82e07cf00 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 12:06:57 +0000 Subject: [PATCH 23/49] Testing: Add role injection step in task definition --- .github/workflows/deploy-reusable.yml | 29 ++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 9be2eab..49011a1 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -169,4 +169,31 @@ jobs: SECRETS_STRING=$(echo "$SECRETS_STRING" | sed '/^$/d') echo "secrets<> $GITHUB_OUTPUT echo "$SECRETS_STRING" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT \ No newline at end of file + echo "EOF" >> $GITHUB_OUTPUT + + - name: Inject roles in task definition + id: inject-roles + run: | + TASK_DEF_PATH="${{ matrix.service.task_def }}" + + if [ ! -f "$TASK_DEF_PATH" ]; then + echo "Task definition not found: $TASK_DEF_PATH - Skipping" + echo "skip=true" >> $GITHUB_OUTPUT + exit 0 + fi + + echo "Injecting roles into task definition: $TASK_DEF_PATH" + + UPDATED_TASK_DEF=$(jq \ + --arg task_role "${{ steps.fetch-secrets.outputs.task_role }}" \ + --arg exec_role "${{ steps.fetch-secrets.outputs.exec_role }}" \ + '.taskRoleArn = $task_role | .executionRoleArn = $exec_role' \ + "$TASK_DEF_PATH") + + echo "Updated task definition:" + echo "$UPDATED_TASK_DEF" | jq '.' + + echo "updated_task_def<> $GITHUB_OUTPUT + echo "$UPDATED_TASK_DEF" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + echo "skip=false" >> $GITHUB_OUTPUT \ No newline at end of file From c79dd791b5d4b15f0999f69b2646038a63d00768 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 12:10:11 +0000 Subject: [PATCH 24/49] Testing: Remove quotes for task definition path in role injection step --- .github/workflows/deploy-reusable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 49011a1..4306a9d 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -174,7 +174,7 @@ jobs: - name: Inject roles in task definition id: inject-roles run: | - TASK_DEF_PATH="${{ matrix.service.task_def }}" + TASK_DEF_PATH=${{ matrix.service.task_def }} if [ ! -f "$TASK_DEF_PATH" ]; then echo "Task definition not found: $TASK_DEF_PATH - Skipping" From 33928f68f9471ef17a20c9618071487e777e75c3 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 12:16:07 +0000 Subject: [PATCH 25/49] Testing: fixed spelling error in lookup --- .github/workflows/deploy-reusable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 4306a9d..a7e0251 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -174,7 +174,7 @@ jobs: - name: Inject roles in task definition id: inject-roles run: | - TASK_DEF_PATH=${{ matrix.service.task_def }} + TASK_DEF_PATH="${{ matrix.service.taskdef }}" if [ ! -f "$TASK_DEF_PATH" ]; then echo "Task definition not found: $TASK_DEF_PATH - Skipping" From 43a36dd5e3bf9938cc3f3b4aeca84d5ab75e59db Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 12:31:25 +0000 Subject: [PATCH 26/49] Testing: Add ECS task definition rendering step with secrets and preview output --- .github/workflows/deploy-reusable.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index a7e0251..c21b72c 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -196,4 +196,20 @@ jobs: echo "updated_task_def<> $GITHUB_OUTPUT echo "$UPDATED_TASK_DEF" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - echo "skip=false" >> $GITHUB_OUTPUT \ No newline at end of file + echo "skip=false" >> $GITHUB_OUTPUT + + - name: Render ECS task definition with secrets + id: render-task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1.8.1 + with: + task-definition: ${{ steps.inject-roles.outputs.updated_task_def }} + container-name: ${{ matrix.service.container }} + # image: ${{ steps.build-image.outputs.IMAGE }} + environment-variables: | + DOTNET_ENVIRONMENT=${{ inputs.environment == 'prod' && 'Production' || 'PreProduction' }} + secrets: ${{ steps.build-secrets.outputs.secrets }} + + - name: Preview rendered ECS task definition + run: | + echo "Rendered task definition:" + echo "${{ steps.render-task-def.outputs.task-definition }}" | jq '.' \ No newline at end of file From 8010a4bda765a1a4413fde56a8114840a1ec845a Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 12:33:34 +0000 Subject: [PATCH 27/49] Testing: Add image parameter for ECS task definition (required) --- .github/workflows/deploy-reusable.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index c21b72c..899d816 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -204,6 +204,7 @@ jobs: with: task-definition: ${{ steps.inject-roles.outputs.updated_task_def }} container-name: ${{ matrix.service.container }} + image: amazon/amazon-ecs-sample:latest # image: ${{ steps.build-image.outputs.IMAGE }} environment-variables: | DOTNET_ENVIRONMENT=${{ inputs.environment == 'prod' && 'Production' || 'PreProduction' }} From 15154b3eb7724f9b85ff3ea2a4bb6f76d21b166a Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 12:39:05 +0000 Subject: [PATCH 28/49] Testing: Saving task def to file so it can be rendered correctly --- .github/workflows/deploy-reusable.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 899d816..ed3b3c4 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -190,19 +190,20 @@ jobs: '.taskRoleArn = $task_role | .executionRoleArn = $exec_role' \ "$TASK_DEF_PATH") + UPDATED_TASK_DEF_FILE="rendered-task-def.json" + echo "$UPDATED_TASK_DEF" > "$UPDATED_TASK_DEF_FILE" + echo "Updated task definition:" - echo "$UPDATED_TASK_DEF" | jq '.' + cat "$UPDATED_TASK_DEF_FILE" | jq '.' - echo "updated_task_def<> $GITHUB_OUTPUT - echo "$UPDATED_TASK_DEF" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT + echo "updated_task_def_file=$UPDATED_TASK_DEF_FILE" >> $GITHUB_OUTPUT echo "skip=false" >> $GITHUB_OUTPUT - name: Render ECS task definition with secrets id: render-task-def uses: aws-actions/amazon-ecs-render-task-definition@v1.8.1 with: - task-definition: ${{ steps.inject-roles.outputs.updated_task_def }} + task-definition: ${{ steps.inject-roles.outputs.updated_task_def_file }} container-name: ${{ matrix.service.container }} image: amazon/amazon-ecs-sample:latest # image: ${{ steps.build-image.outputs.IMAGE }} From 840e1a9f5d2aac3a0fa8ac1d7947d3ce3312af02 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 12:41:28 +0000 Subject: [PATCH 29/49] Testing: Update ECS task definition preview step to use updated task definition file --- .github/workflows/deploy-reusable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index ed3b3c4..82de624 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -214,4 +214,4 @@ jobs: - name: Preview rendered ECS task definition run: | echo "Rendered task definition:" - echo "${{ steps.render-task-def.outputs.task-definition }}" | jq '.' \ No newline at end of file + cat "${{ steps.inject-roles.outputs.updated_task_def_file }}" | jq '.' \ No newline at end of file From ce018a231cbfadc80055fe5fb9c6779338d07168 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 12:50:07 +0000 Subject: [PATCH 30/49] Testing: Enable test execution and image publishing in deployment workflow --- .github/workflows/deploy-reusable.yml | 69 ++++++++++++++------------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 82de624..bbf7f58 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -81,37 +81,37 @@ jobs: with: dotnet-version: 10.0.x - # - name: Run tests - # run: dotnet test --configuration Release - - # - name: Build and Publish Image - # id: build-image - # env: - # REGISTRY: ${{ steps.login-ecr.outputs.registry }} - # REPOSITORY: dms-${{ inputs.environment }}-apps - # IMAGE_TAG: ${{ matrix.service.name }}-${{ github.sha }} - # run: | - # PROJECT_PATH="src/${{ matrix.service.project }}" + - name: Run tests + run: dotnet test --configuration Release + + - name: Build and Publish Image + id: build-image + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: dms-${{ inputs.environment }}-apps + IMAGE_TAG: ${{ matrix.service.name }}-${{ github.sha }} + run: | + PROJECT_PATH="src/${{ matrix.service.project }}" - # if [ ! -f "$PROJECT_PATH" ]; then - # echo "Project not found: $PROJECT_PATH - Skipping" - # echo "skip=true" >> $GITHUB_OUTPUT - # exit 0 - # fi + if [ ! -f "$PROJECT_PATH" ]; then + echo "Project not found: $PROJECT_PATH - Skipping" + echo "skip=true" >> $GITHUB_OUTPUT + exit 0 + fi - # echo "Building and publishing: $PROJECT_PATH" + echo "Building and publishing: $PROJECT_PATH" - # dotnet publish "$PROJECT_PATH" \ - # --configuration Release \ - # --target:PublishContainer \ - # --property:ContainerRegistry=$REGISTRY \ - # --property:ContainerRepository=$REPOSITORY \ - # --property:ContainerImageTag=$IMAGE_TAG + dotnet publish "$PROJECT_PATH" \ + --configuration Release \ + --target:PublishContainer \ + --property:ContainerRegistry=$REGISTRY \ + --property:ContainerRepository=$REPOSITORY \ + --property:ContainerImageTag=$IMAGE_TAG - # IMAGE="${REGISTRY}/${REPOSITORY}:${IMAGE_TAG}" - # echo "IMAGE=${IMAGE}" >> $GITHUB_OUTPUT - # echo "skip=false" >> $GITHUB_OUTPUT - # echo "Published image: ${IMAGE}" + IMAGE="${REGISTRY}/${REPOSITORY}:${IMAGE_TAG}" + echo "IMAGE=${IMAGE}" >> $GITHUB_OUTPUT + echo "skip=false" >> $GITHUB_OUTPUT + echo "Published image: ${IMAGE}" - name: Testing workflow steps run: echo "The testing / deployment of the image have been skipped for this demo, but the workflow is correctly iterating over the services and would deploy the built image to ECS if those steps were uncommented." @@ -205,13 +205,16 @@ jobs: with: task-definition: ${{ steps.inject-roles.outputs.updated_task_def_file }} container-name: ${{ matrix.service.container }} - image: amazon/amazon-ecs-sample:latest - # image: ${{ steps.build-image.outputs.IMAGE }} + image: ${{ steps.build-image.outputs.IMAGE }} environment-variables: | DOTNET_ENVIRONMENT=${{ inputs.environment == 'prod' && 'Production' || 'PreProduction' }} secrets: ${{ steps.build-secrets.outputs.secrets }} - - name: Preview rendered ECS task definition - run: | - echo "Rendered task definition:" - cat "${{ steps.inject-roles.outputs.updated_task_def_file }}" | jq '.' \ No newline at end of file + - name: Deploy ECS service + if: steps.build-image.outputs.skip != 'true' + uses: aws-actions/amazon-ecs-deploy-task-definition@v2 + with: + task-definition: ${{ steps.render-task-def.outputs.task-definition }} + service: DMS-${{ inputs.environment == 'prod' && 'Prod' || 'PreProd' }}-${{ matrix.service.name }}-service + cluster: DMS-${{ inputs.environment == 'prod' && 'Prod' || 'PreProd' }}-Cluster + wait-for-service-stability: true \ No newline at end of file From 0ebf7d4cbccf54a8f14f702fa84263fc4deac791 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 13:06:15 +0000 Subject: [PATCH 31/49] Testing: Update secret handling and task definition processing in deployment workflow --- .github/workflows/deploy-reusable.yml | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index bbf7f58..cbf18e5 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -156,10 +156,13 @@ jobs: ["Sentry_Dsn"]="Sentry_Dsn" ) + # Get the full secret ARN + SECRET_ARN="${{ secrets.AWS_SECRETS_ARN }}" + for secret in $(echo "$SECRETS_JSON" | jq -r '.[]'); do env_var_name="${SECRET_MAP[$secret]}" if [ -n "$env_var_name" ]; then - SECRETS_STRING+="${env_var_name}=\${{ secrets.AWS_SECRETS_ARN }}:${secret}::"$'\n' + SECRETS_STRING+="${env_var_name}=${SECRET_ARN}:${secret}::"$'\n' fi done @@ -171,10 +174,10 @@ jobs: echo "$SECRETS_STRING" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - - name: Inject roles in task definition + - name: Replace EFS placeholders and inject roles in task definition id: inject-roles run: | - TASK_DEF_PATH="${{ matrix.service.taskdef }}" + TASK_DEF_PATH="${{ matrix.service.taskDef }}" if [ ! -f "$TASK_DEF_PATH" ]; then echo "Task definition not found: $TASK_DEF_PATH - Skipping" @@ -182,8 +185,15 @@ jobs: exit 0 fi - echo "Injecting roles into task definition: $TASK_DEF_PATH" - + echo "Processing task definition: $TASK_DEF_PATH" + + # Step 1: Replace EFS placeholders + sed -i.bak \ + -e 's/EFS_FILE_SYSTEM_ID/${{ steps.fetch-secrets.outputs.efs-fs-id }}/g' \ + -e 's/EFS_ACCESS_POINT_ID/${{ steps.fetch-secrets.outputs.efs-ap-id }}/g' \ + "$TASK_DEF_PATH" + + # Step 2: Inject IAM roles UPDATED_TASK_DEF=$(jq \ --arg task_role "${{ steps.fetch-secrets.outputs.task_role }}" \ --arg exec_role "${{ steps.fetch-secrets.outputs.exec_role }}" \ From ddbb4ef82d9f5da0f5e04734b8e9a2799df1c4c7 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 13:46:56 +0000 Subject: [PATCH 32/49] Testing: Enhance deployment workflow with EFS parameters and update task definitions to use placeholders --- .github/workflows/deploy-reusable.yml | 11 +- infra/task-definitions/api-task-def.json | 92 ++++++------ infra/task-definitions/blocking-task-def.json | 2 +- infra/task-definitions/cleanup-task-def.json | 2 +- .../dbinteractions-task-def.json | 2 +- .../delius-parser-task-def.json | 2 +- infra/task-definitions/filesync-task-def.json | 2 +- infra/task-definitions/import-task-def.json | 2 +- infra/task-definitions/logging-task-def.json | 2 +- .../matching-engine-task-def.json | 2 +- infra/task-definitions/meow-task-def.json | 2 +- .../offloc-cleaner-task-def.json | 2 +- .../offloc-parser-task-def.json | 2 +- .../task-definitions/visualiser-task-def.json | 142 +++++++++--------- 14 files changed, 137 insertions(+), 130 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index cbf18e5..6b6ec56 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -126,9 +126,13 @@ jobs: TASK_ROLE=$(echo "$SECRET_JSON" | jq -r '.Task_Role_ARN') EXEC_ROLE=$(echo "$SECRET_JSON" | jq -r '.Execution_Role_ARN') + EFS_FS_ID=$(echo "$SECRET_JSON" | jq -r '.EFS_File_System_ID') + EFS_AP_ID=$(echo "$SECRET_JSON" | jq -r '.EFS_Access_Point_ID') echo "task_role=$TASK_ROLE" >> $GITHUB_OUTPUT echo "exec_role=$EXEC_ROLE" >> $GITHUB_OUTPUT + echo "efs-fs-id=$EFS_FS_ID" >> $GITHUB_OUTPUT + echo "efs-ap-id=$EFS_AP_ID" >> $GITHUB_OUTPUT - name: Build secrets list for service id: build-secrets @@ -193,11 +197,14 @@ jobs: -e 's/EFS_ACCESS_POINT_ID/${{ steps.fetch-secrets.outputs.efs-ap-id }}/g' \ "$TASK_DEF_PATH" - # Step 2: Inject IAM roles + # Step 2: Inject IAM roles and set family name to match Terraform convention + FAMILY_NAME="DMS-${{ inputs.environment == 'prod' && 'Prod' || 'PreProd' }}-${{ matrix.service.name }}-task" + UPDATED_TASK_DEF=$(jq \ --arg task_role "${{ steps.fetch-secrets.outputs.task_role }}" \ --arg exec_role "${{ steps.fetch-secrets.outputs.exec_role }}" \ - '.taskRoleArn = $task_role | .executionRoleArn = $exec_role' \ + --arg family "$FAMILY_NAME" \ + '.taskRoleArn = $task_role | .executionRoleArn = $exec_role | .family = $family' \ "$TASK_DEF_PATH") UPDATED_TASK_DEF_FILE="rendered-task-def.json" diff --git a/infra/task-definitions/api-task-def.json b/infra/task-definitions/api-task-def.json index 995e44f..1bf493f 100644 --- a/infra/task-definitions/api-task-def.json +++ b/infra/task-definitions/api-task-def.json @@ -1,50 +1,50 @@ { - "containerDefinitions": [ + "containerDefinitions": [ + { + "name": "api-container", + "image": "PLACEHOLDER", + "cpu": 512, + "memory": 1024, + "portMappings": [ { - "name": "api-container", - "image": "PLACEHOLDER", - "cpu": 512, - "memory": 1024, - "portMappings": [ - { - "containerPort": 8080, - "hostPort": 8080, - "protocol": "tcp", - "name": "http" - }, - { - "containerPort": 8081, - "hostPort": 8081, - "protocol": "tcp", - "name": "https" - } - ], - "essential": true, - "environment": [], - "secrets": [], - "mountPoints": [], - "volumesFrom": [], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "/ecs/dms-api", - "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" - } - }, - "systemControls": [] + "containerPort": 8080, + "hostPort": 8080, + "protocol": "tcp", + "name": "http" + }, + { + "containerPort": 8081, + "hostPort": 8081, + "protocol": "tcp", + "name": "https" + } + ], + "essential": true, + "environment": [], + "secrets": [], + "mountPoints": [], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/dms-api", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" } - ], - "family": "dms-api-task", - "networkMode": "awsvpc", - "volumes": [], - "placementConstraints": [], - "compatibilities": [ - "FARGATE" - ], - "requiresCompatibilities": [ - "FARGATE" - ], - "cpu": "512", - "memory": "1024" + }, + "systemControls": [] + } + ], + "family": "PLACEHOLDER", + "networkMode": "awsvpc", + "volumes": [], + "placementConstraints": [], + "compatibilities": [ + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "512", + "memory": "1024" } diff --git a/infra/task-definitions/blocking-task-def.json b/infra/task-definitions/blocking-task-def.json index 259e225..b01ca18 100644 --- a/infra/task-definitions/blocking-task-def.json +++ b/infra/task-definitions/blocking-task-def.json @@ -27,7 +27,7 @@ "systemControls": [] } ], - "family": "dms-blocking-task", + "family": "PLACEHOLDER", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/cleanup-task-def.json b/infra/task-definitions/cleanup-task-def.json index 813db55..67bb6d7 100644 --- a/infra/task-definitions/cleanup-task-def.json +++ b/infra/task-definitions/cleanup-task-def.json @@ -27,7 +27,7 @@ "systemControls": [] } ], - "family": "dms-cleanup-task", + "family": "PLACEHOLDER", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/dbinteractions-task-def.json b/infra/task-definitions/dbinteractions-task-def.json index b6d0d2a..ee7dd4d 100644 --- a/infra/task-definitions/dbinteractions-task-def.json +++ b/infra/task-definitions/dbinteractions-task-def.json @@ -27,7 +27,7 @@ "systemControls": [] } ], - "family": "dms-dbinteractions-task", + "family": "PLACEHOLDER", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/delius-parser-task-def.json b/infra/task-definitions/delius-parser-task-def.json index fa000b1..faf1762 100644 --- a/infra/task-definitions/delius-parser-task-def.json +++ b/infra/task-definitions/delius-parser-task-def.json @@ -27,7 +27,7 @@ "systemControls": [] } ], - "family": "dms-delius-parser-task", + "family": "PLACEHOLDER", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/filesync-task-def.json b/infra/task-definitions/filesync-task-def.json index 1ba6672..e37584a 100644 --- a/infra/task-definitions/filesync-task-def.json +++ b/infra/task-definitions/filesync-task-def.json @@ -27,7 +27,7 @@ "systemControls": [] } ], - "family": "dms-filesync-task", + "family": "PLACEHOLDER", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/import-task-def.json b/infra/task-definitions/import-task-def.json index 6b1c827..403e0d3 100644 --- a/infra/task-definitions/import-task-def.json +++ b/infra/task-definitions/import-task-def.json @@ -27,7 +27,7 @@ "systemControls": [] } ], - "family": "dms-import-task", + "family": "PLACEHOLDER", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/logging-task-def.json b/infra/task-definitions/logging-task-def.json index f977aa1..a30d44b 100644 --- a/infra/task-definitions/logging-task-def.json +++ b/infra/task-definitions/logging-task-def.json @@ -27,7 +27,7 @@ "systemControls": [] } ], - "family": "dms-logging-task", + "family": "PLACEHOLDER", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/matching-engine-task-def.json b/infra/task-definitions/matching-engine-task-def.json index e7fe8d7..8fc54e7 100644 --- a/infra/task-definitions/matching-engine-task-def.json +++ b/infra/task-definitions/matching-engine-task-def.json @@ -27,7 +27,7 @@ "systemControls": [] } ], - "family": "dms-matching-engine-task", + "family": "PLACEHOLDER", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/meow-task-def.json b/infra/task-definitions/meow-task-def.json index 50cd3a3..36fe49e 100644 --- a/infra/task-definitions/meow-task-def.json +++ b/infra/task-definitions/meow-task-def.json @@ -27,7 +27,7 @@ "systemControls": [] } ], - "family": "dms-meow-task", + "family": "PLACEHOLDER", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/offloc-cleaner-task-def.json b/infra/task-definitions/offloc-cleaner-task-def.json index 75b81b3..0d74f73 100644 --- a/infra/task-definitions/offloc-cleaner-task-def.json +++ b/infra/task-definitions/offloc-cleaner-task-def.json @@ -27,7 +27,7 @@ "systemControls": [] } ], - "family": "dms-offloc-cleaner-task", + "family": "PLACEHOLDER", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/offloc-parser-task-def.json b/infra/task-definitions/offloc-parser-task-def.json index a8aa686..05d5164 100644 --- a/infra/task-definitions/offloc-parser-task-def.json +++ b/infra/task-definitions/offloc-parser-task-def.json @@ -27,7 +27,7 @@ "systemControls": [] } ], - "family": "dms-offloc-parser-task", + "family": "PLACEHOLDER", "networkMode": "awsvpc", "volumes": [ { diff --git a/infra/task-definitions/visualiser-task-def.json b/infra/task-definitions/visualiser-task-def.json index b857867..2fce19d 100644 --- a/infra/task-definitions/visualiser-task-def.json +++ b/infra/task-definitions/visualiser-task-def.json @@ -1,76 +1,76 @@ { - "containerDefinitions": [ + "containerDefinitions": [ + { + "name": "visualiser-container", + "image": "PLACEHOLDER", + "cpu": 256, + "memory": 512, + "portMappings": [ { - "name": "visualiser-container", - "image": "PLACEHOLDER", - "cpu": 256, - "memory": 512, - "portMappings": [ - { - "containerPort": 8080, - "hostPort": 8080, - "protocol": "tcp", - "name": "http" - }, - { - "containerPort": 8081, - "hostPort": 8081, - "protocol": "tcp", - "name": "https" - } - ], - "essential": true, - "environment": [], - "mountPoints": [], - "volumesFrom": [], - "logConfiguration": { - "logDriver": "awslogs", - "options": { - "awslogs-group": "ecs/visualiser", - "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" - } - }, - "systemControls": [] - } - ], - "family": "visualiser-task", - "networkMode": "awsvpc", - "revision": 6, - "volumes": [], - "status": "ACTIVE", - "requiresAttributes": [ - { - "name": "com.amazonaws.ecs.capability.logging-driver.awslogs" - }, - { - "name": "ecs.capability.execution-role-awslogs" + "containerPort": 8080, + "hostPort": 8080, + "protocol": "tcp", + "name": "http" }, { - "name": "com.amazonaws.ecs.capability.ecr-auth" - }, - { - "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19" - }, - { - "name": "ecs.capability.execution-role-ecr-pull" - }, - { - "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18" - }, - { - "name": "ecs.capability.task-eni" + "containerPort": 8081, + "hostPort": 8081, + "protocol": "tcp", + "name": "https" + } + ], + "essential": true, + "environment": [], + "mountPoints": [], + "volumesFrom": [], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "ecs/visualiser", + "awslogs-region": "eu-west-2", + "awslogs-stream-prefix": "ecs" } - ], - "placementConstraints": [], - "compatibilities": [ - "EC2", - "MANAGED_INSTANCES", - "FARGATE" - ], - "requiresCompatibilities": [ - "FARGATE" - ], - "cpu": "256", - "memory": "512" -} \ No newline at end of file + }, + "systemControls": [] + } + ], + "family": "PLACEHOLDER", + "networkMode": "awsvpc", + "revision": 6, + "volumes": [], + "status": "ACTIVE", + "requiresAttributes": [ + { + "name": "com.amazonaws.ecs.capability.logging-driver.awslogs" + }, + { + "name": "ecs.capability.execution-role-awslogs" + }, + { + "name": "com.amazonaws.ecs.capability.ecr-auth" + }, + { + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19" + }, + { + "name": "ecs.capability.execution-role-ecr-pull" + }, + { + "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18" + }, + { + "name": "ecs.capability.task-eni" + } + ], + "placementConstraints": [], + "compatibilities": [ + "EC2", + "MANAGED_INSTANCES", + "FARGATE" + ], + "requiresCompatibilities": [ + "FARGATE" + ], + "cpu": "256", + "memory": "512" +} From 04a5811c307598c785f70105ba96fdb6cd4dd380 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 18:07:44 +0000 Subject: [PATCH 33/49] Fix: Update family name and service/cluster naming conventions to lowercase in deployment workflow --- .github/workflows/deploy-reusable.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 6b6ec56..f268afa 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -198,7 +198,7 @@ jobs: "$TASK_DEF_PATH" # Step 2: Inject IAM roles and set family name to match Terraform convention - FAMILY_NAME="DMS-${{ inputs.environment == 'prod' && 'Prod' || 'PreProd' }}-${{ matrix.service.name }}-task" + FAMILY_NAME="dms-${{ inputs.environment == 'prod' && 'prod' || 'preprod' }}-${{ matrix.service.name }}-task" UPDATED_TASK_DEF=$(jq \ --arg task_role "${{ steps.fetch-secrets.outputs.task_role }}" \ @@ -232,6 +232,6 @@ jobs: uses: aws-actions/amazon-ecs-deploy-task-definition@v2 with: task-definition: ${{ steps.render-task-def.outputs.task-definition }} - service: DMS-${{ inputs.environment == 'prod' && 'Prod' || 'PreProd' }}-${{ matrix.service.name }}-service - cluster: DMS-${{ inputs.environment == 'prod' && 'Prod' || 'PreProd' }}-Cluster + service: dms-${{ inputs.environment == 'prod' && 'prod' || 'preprod' }}-${{ matrix.service.name }}-service + cluster: dms-${{ inputs.environment == 'prod' && 'prod' || 'preprod' }}-cluster wait-for-service-stability: true \ No newline at end of file From d2886a80929fa6636ded1e7f4fe19d8ddd4cf0a8 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Feb 2026 19:06:50 +0000 Subject: [PATCH 34/49] Fix: Update task definitions to replace placeholders with environment-specific values for logging configuration --- .github/workflows/deploy-reusable.yml | 5 ++++- infra/task-definitions/api-task-def.json | 4 ++-- infra/task-definitions/blocking-task-def.json | 4 ++-- infra/task-definitions/cleanup-task-def.json | 4 ++-- infra/task-definitions/dbinteractions-task-def.json | 4 ++-- infra/task-definitions/delius-parser-task-def.json | 4 ++-- infra/task-definitions/filesync-task-def.json | 4 ++-- infra/task-definitions/import-task-def.json | 4 ++-- infra/task-definitions/logging-task-def.json | 4 ++-- infra/task-definitions/matching-engine-task-def.json | 4 ++-- infra/task-definitions/meow-task-def.json | 4 ++-- infra/task-definitions/offloc-cleaner-task-def.json | 4 ++-- infra/task-definitions/offloc-parser-task-def.json | 4 ++-- infra/task-definitions/visualiser-task-def.json | 4 ++-- 14 files changed, 30 insertions(+), 27 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index f268afa..34e95c1 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -191,10 +191,13 @@ jobs: echo "Processing task definition: $TASK_DEF_PATH" - # Step 1: Replace EFS placeholders + # Step 1: Replace placeholders (EFS, CloudWatch) + LOG_GROUP_NAME="dms-${{ inputs.environment == 'prod' && 'prod' || 'preprod' }}/ecs" + sed -i.bak \ -e 's/EFS_FILE_SYSTEM_ID/${{ steps.fetch-secrets.outputs.efs-fs-id }}/g' \ -e 's/EFS_ACCESS_POINT_ID/${{ steps.fetch-secrets.outputs.efs-ap-id }}/g' \ + -e "s|PLACEHOLDER|$LOG_GROUP_NAME|g" \ "$TASK_DEF_PATH" # Step 2: Inject IAM roles and set family name to match Terraform convention diff --git a/infra/task-definitions/api-task-def.json b/infra/task-definitions/api-task-def.json index 1bf493f..aed85c3 100644 --- a/infra/task-definitions/api-task-def.json +++ b/infra/task-definitions/api-task-def.json @@ -27,9 +27,9 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "/ecs/dms-api", + "awslogs-group": "PLACEHOLDER", "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" + "awslogs-stream-prefix": "ecs/api" } }, "systemControls": [] diff --git a/infra/task-definitions/blocking-task-def.json b/infra/task-definitions/blocking-task-def.json index b01ca18..1166d80 100644 --- a/infra/task-definitions/blocking-task-def.json +++ b/infra/task-definitions/blocking-task-def.json @@ -19,9 +19,9 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "/ecs/dms-blocking", + "awslogs-group": "PLACEHOLDER", "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" + "awslogs-stream-prefix": "ecs/blocking" } }, "systemControls": [] diff --git a/infra/task-definitions/cleanup-task-def.json b/infra/task-definitions/cleanup-task-def.json index 67bb6d7..ef294ec 100644 --- a/infra/task-definitions/cleanup-task-def.json +++ b/infra/task-definitions/cleanup-task-def.json @@ -19,9 +19,9 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "/ecs/dms-cleanup", + "awslogs-group": "PLACEHOLDER", "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" + "awslogs-stream-prefix": "ecs/cleanup" } }, "systemControls": [] diff --git a/infra/task-definitions/dbinteractions-task-def.json b/infra/task-definitions/dbinteractions-task-def.json index ee7dd4d..d558a7d 100644 --- a/infra/task-definitions/dbinteractions-task-def.json +++ b/infra/task-definitions/dbinteractions-task-def.json @@ -19,9 +19,9 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "/ecs/dms-dbinteractions", + "awslogs-group": "PLACEHOLDER", "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" + "awslogs-stream-prefix": "ecs/dbinteractions" } }, "systemControls": [] diff --git a/infra/task-definitions/delius-parser-task-def.json b/infra/task-definitions/delius-parser-task-def.json index faf1762..798bae4 100644 --- a/infra/task-definitions/delius-parser-task-def.json +++ b/infra/task-definitions/delius-parser-task-def.json @@ -19,9 +19,9 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "/ecs/dms-delius-parser", + "awslogs-group": "PLACEHOLDER", "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" + "awslogs-stream-prefix": "ecs/delius-parser" } }, "systemControls": [] diff --git a/infra/task-definitions/filesync-task-def.json b/infra/task-definitions/filesync-task-def.json index e37584a..d125180 100644 --- a/infra/task-definitions/filesync-task-def.json +++ b/infra/task-definitions/filesync-task-def.json @@ -19,9 +19,9 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "/ecs/dms-filesync", + "awslogs-group": "PLACEHOLDER", "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" + "awslogs-stream-prefix": "ecs/filesync" } }, "systemControls": [] diff --git a/infra/task-definitions/import-task-def.json b/infra/task-definitions/import-task-def.json index 403e0d3..72eb221 100644 --- a/infra/task-definitions/import-task-def.json +++ b/infra/task-definitions/import-task-def.json @@ -19,9 +19,9 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "/ecs/dms-import", + "awslogs-group": "PLACEHOLDER", "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" + "awslogs-stream-prefix": "ecs/import" } }, "systemControls": [] diff --git a/infra/task-definitions/logging-task-def.json b/infra/task-definitions/logging-task-def.json index a30d44b..c10360b 100644 --- a/infra/task-definitions/logging-task-def.json +++ b/infra/task-definitions/logging-task-def.json @@ -19,9 +19,9 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "/ecs/dms-logging", + "awslogs-group": "PLACEHOLDER", "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" + "awslogs-stream-prefix": "ecs/logging" } }, "systemControls": [] diff --git a/infra/task-definitions/matching-engine-task-def.json b/infra/task-definitions/matching-engine-task-def.json index 8fc54e7..9d59eb6 100644 --- a/infra/task-definitions/matching-engine-task-def.json +++ b/infra/task-definitions/matching-engine-task-def.json @@ -19,9 +19,9 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "/ecs/dms-matching-engine", + "awslogs-group": "PLACEHOLDER", "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" + "awslogs-stream-prefix": "ecs/matching-engine" } }, "systemControls": [] diff --git a/infra/task-definitions/meow-task-def.json b/infra/task-definitions/meow-task-def.json index 36fe49e..2272a18 100644 --- a/infra/task-definitions/meow-task-def.json +++ b/infra/task-definitions/meow-task-def.json @@ -19,9 +19,9 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "/ecs/dms-meow", + "awslogs-group": "PLACEHOLDER", "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" + "awslogs-stream-prefix": "ecs/meow" } }, "systemControls": [] diff --git a/infra/task-definitions/offloc-cleaner-task-def.json b/infra/task-definitions/offloc-cleaner-task-def.json index 0d74f73..b0d3cb4 100644 --- a/infra/task-definitions/offloc-cleaner-task-def.json +++ b/infra/task-definitions/offloc-cleaner-task-def.json @@ -19,9 +19,9 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "/ecs/dms-offloc-cleaner", + "awslogs-group": "PLACEHOLDER", "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" + "awslogs-stream-prefix": "ecs/offloc-cleaner" } }, "systemControls": [] diff --git a/infra/task-definitions/offloc-parser-task-def.json b/infra/task-definitions/offloc-parser-task-def.json index 05d5164..d19f8bd 100644 --- a/infra/task-definitions/offloc-parser-task-def.json +++ b/infra/task-definitions/offloc-parser-task-def.json @@ -19,9 +19,9 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "/ecs/dms-offloc-parser", + "awslogs-group": "PLACEHOLDER", "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" + "awslogs-stream-prefix": "ecs/offloc-parser" } }, "systemControls": [] diff --git a/infra/task-definitions/visualiser-task-def.json b/infra/task-definitions/visualiser-task-def.json index 2fce19d..140e0a2 100644 --- a/infra/task-definitions/visualiser-task-def.json +++ b/infra/task-definitions/visualiser-task-def.json @@ -26,9 +26,9 @@ "logConfiguration": { "logDriver": "awslogs", "options": { - "awslogs-group": "ecs/visualiser", + "awslogs-group": "PLACEHOLDER", "awslogs-region": "eu-west-2", - "awslogs-stream-prefix": "ecs" + "awslogs-stream-prefix": "ecs/visualiser" } }, "systemControls": [] From 44f60d45d5404c2c7466d406b9fac4693a6713ed Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Fri, 27 Feb 2026 11:58:42 +0000 Subject: [PATCH 35/49] Fix: Ensure directories are created for delius and offloc inputs/outputs during FileLocations initialisation --- src/Libraries/EnvironmentSetup/FileLocations.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Libraries/EnvironmentSetup/FileLocations.cs b/src/Libraries/EnvironmentSetup/FileLocations.cs index 6ff2225..e85ff1e 100644 --- a/src/Libraries/EnvironmentSetup/FileLocations.cs +++ b/src/Libraries/EnvironmentSetup/FileLocations.cs @@ -18,6 +18,11 @@ public class FileLocations : IFileLocations public FileLocations(string basePath) { this._basePath = basePath.Replace("~", Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)); + + Directory.CreateDirectory(deliusInput); + Directory.CreateDirectory(deliusOutput); + Directory.CreateDirectory(offlocInput); + Directory.CreateDirectory(offlocOutput); } public string basePath { get => _basePath; } From d1df35e36724ab91af7de21e1864d544cf775838 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 9 Mar 2026 10:37:25 +0000 Subject: [PATCH 36/49] Temp: Stop filesync from processing files entriely --- src/FileSync/appsettings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FileSync/appsettings.json b/src/FileSync/appsettings.json index 4cdb7bd..6943823 100644 --- a/src/FileSync/appsettings.json +++ b/src/FileSync/appsettings.json @@ -24,8 +24,8 @@ }, "SyncOptions": { "ProcessOnStartup": false, - "ProcessOnCompletion": true, - "ProcessOnTimer": true, + "ProcessOnCompletion": false, + "ProcessOnTimer": false, "ProcessTimerIntervalSeconds": 900, "AllowProcessingOlderFiles": false }, From c42d207b7d34cee2c9a7a826ce83170485db8fca Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 9 Mar 2026 14:14:24 +0000 Subject: [PATCH 37/49] Fix: Add API_Client_ID to secrets in services configuration. Previously set to Visualiser Client ID incorrectly causing audience conflict --- .github/workflows/deploy-reusable.yml | 1 + infra/services.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 34e95c1..3335389 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -146,6 +146,7 @@ jobs: ["Authentication_ApiKey"]="Authentication__ApiKey" ["S3_BucketName"]="AWS__S3__BucketName" ["Client_ID"]="AzureAd__ClientId" + ["API_Client_ID"]="AzureAd__ClientId" ["Client_Secret"]="AzureAd__ClientSecret" ["AuditDb"]="ConnectionStrings__AuditDb" ["ClusterDb"]="ConnectionStrings__ClusterDb" diff --git a/infra/services.json b/infra/services.json index 557bc39..458ae1c 100644 --- a/infra/services.json +++ b/infra/services.json @@ -10,7 +10,7 @@ "DeliusRunningPictureDb", "OfflocRunningPictureDb", "ClusterDb", - "Client_ID", + "API_Client_ID", "Authentication_ApiKey", "Sentry_Dsn" ] From 29186a8b71c6bd31cf272512da260f18cc44da34 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 23 Mar 2026 10:41:49 +0000 Subject: [PATCH 38/49] Config: Enable processing on completion and timer in FileSync configuration --- src/FileSync/appsettings.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FileSync/appsettings.json b/src/FileSync/appsettings.json index 6943823..4cdb7bd 100644 --- a/src/FileSync/appsettings.json +++ b/src/FileSync/appsettings.json @@ -24,8 +24,8 @@ }, "SyncOptions": { "ProcessOnStartup": false, - "ProcessOnCompletion": false, - "ProcessOnTimer": false, + "ProcessOnCompletion": true, + "ProcessOnTimer": true, "ProcessTimerIntervalSeconds": 900, "AllowProcessingOlderFiles": false }, From 82e8bd6bf55a58bdd57cb014d4e963a8e99f58ec Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Mon, 30 Mar 2026 13:43:35 +0100 Subject: [PATCH 39/49] Change: Updated gihub actions to use full SHA instead of tags as required by MOJ --- .github/workflows/deploy-reusable.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 3335389..3192b42 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -42,7 +42,7 @@ jobs: services: ${{ steps.load.outputs.services }} steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Load service mappings id: load @@ -64,20 +64,20 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0 with: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} aws-region: ${{ inputs.aws-region }} - name: Login to Amazon ECR id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 + uses: aws-actions/amazon-ecr-login@183a1442edf41672e66566b7fc560e297a290896 # v2.1.1 - name: Setup .NET - uses: actions/setup-dotnet@v5 + uses: actions/setup-dotnet@c2fa09f4bde5ebb9d1777cf28262a3eb3db3ced7 # v5.2.0 with: dotnet-version: 10.0.x @@ -222,7 +222,7 @@ jobs: - name: Render ECS task definition with secrets id: render-task-def - uses: aws-actions/amazon-ecs-render-task-definition@v1.8.1 + uses: aws-actions/amazon-ecs-render-task-definition@77954e213ba1f9f9cb016b86a1d4f6fcdea0d57e # v1.8.4 with: task-definition: ${{ steps.inject-roles.outputs.updated_task_def_file }} container-name: ${{ matrix.service.container }} @@ -233,7 +233,7 @@ jobs: - name: Deploy ECS service if: steps.build-image.outputs.skip != 'true' - uses: aws-actions/amazon-ecs-deploy-task-definition@v2 + uses: aws-actions/amazon-ecs-deploy-task-definition@fc8fc60f3a60ffd500fcb13b209c59d221ac8c8c # v2.6.1 with: task-definition: ${{ steps.render-task-def.outputs.task-definition }} service: dms-${{ inputs.environment == 'prod' && 'prod' || 'preprod' }}-${{ matrix.service.name }}-service From c6edbd3713cb7176f35f770618ee89b35a8accfe Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Wed, 1 Apr 2026 18:22:23 +0100 Subject: [PATCH 40/49] Refactor: Update Offloc staging procedure to use SqlBulkCopy for file imports due to BULK INSERT not working with containerisation Update: Altered FileSync ProcessOnStartup to true --- .../Stored Procedures/Import.sql | 62 ++-------------- .../Services/DbInteractionService.cs | 73 +++++++++++++++++-- src/FileSync/appsettings.json | 2 +- 3 files changed, 72 insertions(+), 65 deletions(-) diff --git a/src/Database/OfflocStagingDb/Stored Procedures/Import.sql b/src/Database/OfflocStagingDb/Stored Procedures/Import.sql index e73abb1..df79ccc 100644 --- a/src/Database/OfflocStagingDb/Stored Procedures/Import.sql +++ b/src/Database/OfflocStagingDb/Stored Procedures/Import.sql @@ -2,75 +2,23 @@ -- Author: Daniel Fenelon -- Create date: 18/03/2024 -- Description: Import of the OffLoc files +-- Note: File loading removed - data is now staged via SqlBulkCopy +-- in the DbInteractions service. This procedure handles +-- post-load standardisation and status update only. -- ============================================= CREATE PROCEDURE [OfflocStaging].[Import] - @basePath VARCHAR(100), @processedFile VARCHAR(50) AS BEGIN SET NOCOUNT ON; SET DATEFORMAT DMY; - - CREATE TABLE #TableNames - ( - Id INT PRIMARY KEY IDENTITY, - fileName VARCHAR(50) NOT NULL - ); - INSERT INTO #TableNames - ( - fileName - ) - VALUES - ('Activities'), - ('Addresses'), - ('Agencies'), - ('Assessments'), - ('Bookings'), - ('Employment'), - ('Flags'), - ('Identifiers'), - ('IncentiveLevel'), - ('Locations'), - ('MainOffence'), - ('Movements'), - ('OffenderAgencies'), - ('OffenderStatus'), - ('OtherOffences'), - ('PersonalDetails'), - ('PNC'), - ('PreviousPrisonNumbers'), - ('SentenceInformation'), - ('SexOffenders'); - - DECLARE @i INT; - DECLARE @sql VARCHAR(250); - DECLARE @fileName VARCHAR(150); - - SET @i = 1; BEGIN TRANSACTION; BEGIN TRY - WHILE @i <= (SELECT COUNT(*)FROM #TableNames) - BEGIN - SET @fileName = - ( - SELECT fileName FROM #TableNames WHERE Id = @i - ); - SET @sql - = N'BULK INSERT OfflocStaging.' + @fileName + N' FROM ''' + @basePath + @fileName + '.txt''' - + N' WITH (FieldTerminator = ''|'', RowTerminator = ''0x0d0a'', MAXERRORS = 1000)'; - EXEC (@sql); - - SET @i = @i + 1; - END; - DECLARE @stringDate CHAR(8); - SET @stringDate = SUBSTRING(@processedFile, 18, 8); - - --Standardise Offloc Data - Declare @retMessage varchar(500); - EXEC [OfflocStaging].[StandardiseData] @retMessage; + DECLARE @retMessage VARCHAR(500); + EXEC [OfflocStaging].[StandardiseData] @retMessage OUTPUT; UPDATE [OfflocRunningPictureDb].[OfflocRunningPicture].[ProcessedFiles] SET [Status] = 'Imported' diff --git a/src/DbInteractions/Services/DbInteractionService.cs b/src/DbInteractions/Services/DbInteractionService.cs index 6416fdd..69c70cf 100644 --- a/src/DbInteractions/Services/DbInteractionService.cs +++ b/src/DbInteractions/Services/DbInteractionService.cs @@ -192,21 +192,48 @@ public async Task StageDelius(string fileName, string filePath) public async Task StageOffloc(string fileName) { string folderName = fileName.Split('.').First(); + string folderPath = Path.Combine(fileLocations.offlocOutput, folderName); + await messageService.PublishAsync(new StatusUpdateMessage($"Offloc staging started for file {fileName}.")); + var tableNames = new[] + { + "Activities", "Addresses", "Agencies", "Assessments", "Bookings", + "Employment", "Flags", "Identifiers", "IncentiveLevel", "Locations", + "MainOffence", "Movements", "OffenderAgencies", "OffenderStatus", + "OtherOffences", "PersonalDetails", "PNC", "PreviousPrisonNumbers", + "SentenceInformation", "SexOffenders" + }; + var offlocConn = new SqlConnection(configuration.GetConnectionString("OfflocStagingDb")!); using (offlocConn) { await offlocConn.OpenAsync(); - SqlCommand command = new SqlCommand(serverConfig.OfflocStagingProcedure, offlocConn); - command.CommandTimeout = 600; - command.CommandType = CommandType.StoredProcedure; - command.Parameters.AddWithValue("@basePath", $"{fileLocations.offlocOutput}/{folderName}/"); - command.Parameters.AddWithValue("@processedFile", fileName); try { - var result = await command.ExecuteNonQueryAsync(); + foreach (var table in tableNames) + { + var filePath = Path.Combine(folderPath, $"{table}.txt"); + if (!File.Exists(filePath)) continue; + + var columns = await GetTableColumns(offlocConn, "OfflocStaging", table); + var dataTable = ReadPipeDelimitedFile(filePath, columns); + + using var bulkCopy = new SqlBulkCopy(offlocConn) + { + DestinationTableName = $"[OfflocStaging].[{table}]", + BulkCopyTimeout = 600 + }; + await bulkCopy.WriteToServerAsync(dataTable); + } + + // Calls StandardiseData + updates ProcessedFiles + SqlCommand command = new SqlCommand(serverConfig.OfflocStagingProcedure, offlocConn); + command.CommandTimeout = 600; + command.CommandType = CommandType.StoredProcedure; + command.Parameters.AddWithValue("@processedFile", fileName); + await command.ExecuteNonQueryAsync(); } catch (Exception e) { @@ -215,7 +242,39 @@ public async Task StageOffloc(string fileName) } } - await messageService.PublishAsync(new StatusUpdateMessage($"Offloc staging finished for file {fileName}.")); + await messageService.PublishAsync(new StatusUpdateMessage($"Offloc Staging completed.")); + } + + private async Task> GetTableColumns(SqlConnection conn, string schema, string table) + { + var columns = new List(); + using var cmd = new SqlCommand( + "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = @schema AND TABLE_NAME = @table ORDER BY ORDINAL_POSITION", + conn); + cmd.Parameters.AddWithValue("@schema", schema); + cmd.Parameters.AddWithValue("@table", table); + using var reader = await cmd.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + columns.Add(reader.GetString(0)); + return columns; + } + + private DataTable ReadPipeDelimitedFile(string filePath, List columns) + { + var dataTable = new DataTable(); + foreach (var col in columns) + dataTable.Columns.Add(col, typeof(string)); + + foreach (var line in File.ReadLines(filePath)) + { + if (string.IsNullOrWhiteSpace(line)) continue; + var values = line.Split('|'); + var row = dataTable.NewRow(); + for (int i = 0; i < Math.Min(values.Length, columns.Count); i++) + row[i] = values[i]; + dataTable.Rows.Add(row); + } + return dataTable; } //Calls merge and then on completion public async Task StandardiseDeliusStaging() diff --git a/src/FileSync/appsettings.json b/src/FileSync/appsettings.json index 4cdb7bd..879dcf3 100644 --- a/src/FileSync/appsettings.json +++ b/src/FileSync/appsettings.json @@ -23,7 +23,7 @@ "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] }, "SyncOptions": { - "ProcessOnStartup": false, + "ProcessOnStartup": true, "ProcessOnCompletion": true, "ProcessOnTimer": true, "ProcessTimerIntervalSeconds": 900, From 056076657d193053d134b9a77a9f1c8c71429ab9 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Thu, 2 Apr 2026 11:17:55 +0100 Subject: [PATCH 41/49] Testing: Update GetTableColumns and ReadPipeDelimitedFile methods to return column max lengths and enforce length constraints --- .../Services/DbInteractionService.cs | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/DbInteractions/Services/DbInteractionService.cs b/src/DbInteractions/Services/DbInteractionService.cs index 69c70cf..42997dd 100644 --- a/src/DbInteractions/Services/DbInteractionService.cs +++ b/src/DbInteractions/Services/DbInteractionService.cs @@ -245,25 +245,29 @@ public async Task StageOffloc(string fileName) await messageService.PublishAsync(new StatusUpdateMessage($"Offloc Staging completed.")); } - private async Task> GetTableColumns(SqlConnection conn, string schema, string table) + private async Task> GetTableColumns(SqlConnection conn, string schema, string table) { - var columns = new List(); + var columns = new List<(string Name, int? MaxLength)>(); using var cmd = new SqlCommand( - "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = @schema AND TABLE_NAME = @table ORDER BY ORDINAL_POSITION", + "SELECT COLUMN_NAME, CHARACTER_MAXIMUM_LENGTH FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = @schema AND TABLE_NAME = @table ORDER BY ORDINAL_POSITION", conn); cmd.Parameters.AddWithValue("@schema", schema); cmd.Parameters.AddWithValue("@table", table); using var reader = await cmd.ExecuteReaderAsync(); while (await reader.ReadAsync()) - columns.Add(reader.GetString(0)); + { + var name = reader.GetString(0); + var maxLength = reader.IsDBNull(1) ? (int?)null : reader.GetInt32(1); + columns.Add((name, maxLength)); + } return columns; } - private DataTable ReadPipeDelimitedFile(string filePath, List columns) + private DataTable ReadPipeDelimitedFile(string filePath, List<(string Name, int? MaxLength)> columns) { var dataTable = new DataTable(); foreach (var col in columns) - dataTable.Columns.Add(col, typeof(string)); + dataTable.Columns.Add(col.Name, typeof(string)); foreach (var line in File.ReadLines(filePath)) { @@ -271,7 +275,13 @@ private DataTable ReadPipeDelimitedFile(string filePath, List columns) var values = line.Split('|'); var row = dataTable.NewRow(); for (int i = 0; i < Math.Min(values.Length, columns.Count); i++) - row[i] = values[i]; + { + var value = values[i]; + var maxLength = columns[i].MaxLength; + if (maxLength.HasValue && maxLength.Value > 0 && value.Length > maxLength.Value) + value = value[..maxLength.Value]; + row[i] = value; + } dataTable.Rows.Add(row); } return dataTable; From 2fc41d7d68c7ebf3a02253ae0fcfa298760e879b Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Thu, 2 Apr 2026 12:11:22 +0100 Subject: [PATCH 42/49] Testing: Enhance GetTableColumns method to include DataType and update ReadPipeDelimitedFile for type handling --- .../Services/DbInteractionService.cs | 56 +++++++++++++++---- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/src/DbInteractions/Services/DbInteractionService.cs b/src/DbInteractions/Services/DbInteractionService.cs index 42997dd..59766c5 100644 --- a/src/DbInteractions/Services/DbInteractionService.cs +++ b/src/DbInteractions/Services/DbInteractionService.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Options; using System.Data; +using System.Globalization; namespace DbInteractions.Services; @@ -245,11 +246,11 @@ public async Task StageOffloc(string fileName) await messageService.PublishAsync(new StatusUpdateMessage($"Offloc Staging completed.")); } - private async Task> GetTableColumns(SqlConnection conn, string schema, string table) + private async Task> GetTableColumns(SqlConnection conn, string schema, string table) { - var columns = new List<(string Name, int? MaxLength)>(); + var columns = new List<(string Name, int? MaxLength, string DataType)>(); using var cmd = new SqlCommand( - "SELECT COLUMN_NAME, CHARACTER_MAXIMUM_LENGTH FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = @schema AND TABLE_NAME = @table ORDER BY ORDINAL_POSITION", + "SELECT COLUMN_NAME, CHARACTER_MAXIMUM_LENGTH, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = @schema AND TABLE_NAME = @table ORDER BY ORDINAL_POSITION", conn); cmd.Parameters.AddWithValue("@schema", schema); cmd.Parameters.AddWithValue("@table", table); @@ -258,16 +259,28 @@ public async Task StageOffloc(string fileName) { var name = reader.GetString(0); var maxLength = reader.IsDBNull(1) ? (int?)null : reader.GetInt32(1); - columns.Add((name, maxLength)); + var dataType = reader.GetString(2); + columns.Add((name, maxLength, dataType)); } return columns; } - private DataTable ReadPipeDelimitedFile(string filePath, List<(string Name, int? MaxLength)> columns) + private static readonly string[] DateTypes = ["date", "datetime", "datetime2", "smalldatetime"]; + private static readonly string[] IntTypes = ["int", "smallint", "tinyint", "bigint"]; + private static readonly CultureInfo UkCulture = new CultureInfo("en-GB"); + + private DataTable ReadPipeDelimitedFile(string filePath, List<(string Name, int? MaxLength, string DataType)> columns) { var dataTable = new DataTable(); foreach (var col in columns) - dataTable.Columns.Add(col.Name, typeof(string)); + { + if (DateTypes.Contains(col.DataType)) + dataTable.Columns.Add(col.Name, typeof(DateTime)).AllowDBNull = true; + else if (IntTypes.Contains(col.DataType)) + dataTable.Columns.Add(col.Name, typeof(int)).AllowDBNull = true; + else + dataTable.Columns.Add(col.Name, typeof(string)); + } foreach (var line in File.ReadLines(filePath)) { @@ -277,10 +290,33 @@ private DataTable ReadPipeDelimitedFile(string filePath, List<(string Name, int? for (int i = 0; i < Math.Min(values.Length, columns.Count); i++) { var value = values[i]; - var maxLength = columns[i].MaxLength; - if (maxLength.HasValue && maxLength.Value > 0 && value.Length > maxLength.Value) - value = value[..maxLength.Value]; - row[i] = value; + var col = columns[i]; + + if (string.IsNullOrWhiteSpace(value)) + { + row[i] = DateTypes.Contains(col.DataType) || IntTypes.Contains(col.DataType) + ? DBNull.Value + : (object)""; + continue; + } + + if (DateTypes.Contains(col.DataType)) + { + row[i] = DateTime.TryParse(value, UkCulture, DateTimeStyles.None, out var dt) + ? dt + : DBNull.Value; + } + else if (IntTypes.Contains(col.DataType)) + { + row[i] = int.TryParse(value, out var n) ? n : DBNull.Value; + } + else + { + var maxLength = col.MaxLength; + if (maxLength.HasValue && maxLength.Value > 0 && value.Length > maxLength.Value) + value = value[..maxLength.Value]; + row[i] = value; + } } dataTable.Rows.Add(row); } From 7e40395fdc222813af4fe3d11002c263f58c8a50 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Thu, 2 Apr 2026 12:42:40 +0100 Subject: [PATCH 43/49] Testig: Change Delius to also use SqlBulkCopy like Offloc does --- .../Stored Procedures/StageDelius.sql | 44 ++--------- .../Services/DbInteractionService.cs | 76 ++++++++++++++----- 2 files changed, 63 insertions(+), 57 deletions(-) diff --git a/src/Database/DeliusStagingDb/Stored Procedures/StageDelius.sql b/src/Database/DeliusStagingDb/Stored Procedures/StageDelius.sql index df6f73e..7053ae8 100644 --- a/src/Database/DeliusStagingDb/Stored Procedures/StageDelius.sql +++ b/src/Database/DeliusStagingDb/Stored Procedures/StageDelius.sql @@ -1,47 +1,19 @@ CREATE PROCEDURE [DeliusStaging].[StageDelius] - @basePath VARCHAR(100), -- Should be in the format /app/delius/{fileName}. - @processedFile VARCHAR(50), - @inContainer VARCHAR(1) + @processedFile VARCHAR(50) AS - IF @inContainer = 'N' - BEGIN - SET NOCOUNT ON; - SET DATEFORMAT DMY; - END; +BEGIN + SET NOCOUNT ON; + SET DATEFORMAT DMY; - CREATE TABLE #TableNames(Id int Primary Key Identity, fileName VARCHAR(50)); - INSERT INTO #TableNames(fileName) VALUES - ('AdditionalIdentifier'), ('AliasDetails'), ('Disability'), ('Disposal'), ('EventDetails'), ('Header'), ('MainOffence'), - ('OAS'), ('OffenderAddress'), ('OffenderManager'), ('OffenderManagerBuildings'), ('OffenderManagerTeam'), - ('OffenderToOffenderManagerMappings'), ('Offenders'), ('OffenderTransfer'), ('PersonalCircumstances'), - ('Provision'), ('RegistrationDetails'), ('Requirement'); - - DECLARE @i int; - DECLARE @sql NVARCHAR(250); - DECLARE @fileName VARCHAR(150); - - SET @i = 1; - - BEGIN TRANSACTION; + BEGIN TRANSACTION; BEGIN TRY - WHILE @i <= (Select COUNT(*) FROM #TableNames) - BEGIN - SET @fileName = (SELECT fileName FROM #TableNames WHERE Id = @i); - SET @sql = 'BULK INSERT DeliusStaging.' + @fileName + ' FROM ''' + @basePath + @fileName + '.txt''' + - ' WITH (FieldTerminator=''|'', RowTerminator = ''0x0d0a'', MAXERRORS = 1000)'; - - EXEC (@sql); - SET @i = @i+1; - END - - --Standardise Delius Data - Declare @retMessage varchar(500); - EXEC [DeliusStaging].[StandardiseData] @retMessage; + DECLARE @retMessage VARCHAR(500); + EXEC [DeliusStaging].[StandardiseData] @retMessage OUTPUT; UPDATE [DeliusRunningPictureDb].[DeliusRunningPicture].[ProcessedFiles] SET [Status] = 'Imported' WHERE FileName = @processedFile; - + COMMIT TRANSACTION; END TRY BEGIN CATCH diff --git a/src/DbInteractions/Services/DbInteractionService.cs b/src/DbInteractions/Services/DbInteractionService.cs index 59766c5..68d071d 100644 --- a/src/DbInteractions/Services/DbInteractionService.cs +++ b/src/DbInteractions/Services/DbInteractionService.cs @@ -14,7 +14,6 @@ public class DbInteractionService : IDbInteractionService private readonly IFileLocations fileLocations; private readonly ServerConfiguration serverConfig; private readonly IConfiguration configuration; - private readonly bool inContainer; private readonly IMessageService messageService; public DbInteractionService( @@ -27,7 +26,6 @@ public DbInteractionService( this.configuration = config; this.fileLocations = fileLocations; this.messageService = messageService; - this.inContainer = config.GetValue("RUNNING_IN_CONTAINER"); } public async Task GetProcessedDeliusFileNames() @@ -151,34 +149,48 @@ SELECT FileName as [Name] FROM [OfflocRunningPicture].[ProcessedFiles] public async Task StageDelius(string fileName, string filePath) { string folderName = filePath.Split('/').Last(); + string folderPath = Path.Combine(fileLocations.deliusOutput, folderName); await messageService.PublishAsync(new StatusUpdateMessage($"Delius staging started for file number {fileName}")); - string containerFlag = string.Empty; - if (inContainer) - { - containerFlag = "Y"; - } - else + var tableNames = new[] { - containerFlag = "N"; - } + "AdditionalIdentifier", "AliasDetails", "Disability", "Disposal", "EventDetails", + "Header", "MainOffence", "OAS", "OffenderAddress", "OffenderManager", + "OffenderManagerBuildings", "OffenderManagerTeam", "OffenderToOffenderManagerMappings", + "Offenders", "OffenderTransfer", "PersonalCircumstances", "Provision", + "RegistrationDetails", "Requirement" + }; var deliusConn = new SqlConnection(configuration.GetConnectionString("DeliusStagingDb")!); using (deliusConn) { await deliusConn.OpenAsync(); - SqlCommand cmd = new SqlCommand(serverConfig.DeliusStagingProcedure, deliusConn); - cmd.CommandType = CommandType.StoredProcedure; - cmd.CommandTimeout = 600; - cmd.Parameters.AddWithValue("@basePath", $"{fileLocations.deliusOutput}/{folderName}/"); - cmd.Parameters.AddWithValue("@processedFile", fileName); - cmd.Parameters.AddWithValue("@inContainer", containerFlag); - try { - var res = await cmd.ExecuteNonQueryAsync(); + foreach (var table in tableNames) + { + var tableFilePath = Path.Combine(folderPath, $"{table}.txt"); + if (!File.Exists(tableFilePath)) continue; + + var columns = await GetTableColumns(deliusConn, "DeliusStaging", table); + var dataTable = ReadPipeDelimitedFile(tableFilePath, columns); + + using var bulkCopy = new SqlBulkCopy(deliusConn) + { + DestinationTableName = $"[DeliusStaging].[{table}]", + BulkCopyTimeout = 600 + }; + await bulkCopy.WriteToServerAsync(dataTable); + } + + // Calls StandardiseData + updates ProcessedFiles + SqlCommand command = new SqlCommand(serverConfig.DeliusStagingProcedure, deliusConn); + command.CommandType = CommandType.StoredProcedure; + command.CommandTimeout = 600; + command.Parameters.AddWithValue("@processedFile", fileName); + await command.ExecuteNonQueryAsync(); } catch (Exception e) { @@ -187,7 +199,7 @@ public async Task StageDelius(string fileName, string filePath) } } - await messageService.PublishAsync(new StatusUpdateMessage($"Delius staging finished for file {fileName}.")); + await messageService.PublishAsync(new StatusUpdateMessage($"Delius Staging completed.")); } public async Task StageOffloc(string fileName) @@ -266,7 +278,9 @@ public async Task StageOffloc(string fileName) } private static readonly string[] DateTypes = ["date", "datetime", "datetime2", "smalldatetime"]; - private static readonly string[] IntTypes = ["int", "smallint", "tinyint", "bigint"]; + private static readonly string[] IntTypes = ["int", "smallint", "tinyint"]; + private static readonly string[] LongTypes = ["bigint"]; + private static readonly string[] BinaryTypes = ["varbinary", "binary"]; private static readonly CultureInfo UkCulture = new CultureInfo("en-GB"); private DataTable ReadPipeDelimitedFile(string filePath, List<(string Name, int? MaxLength, string DataType)> columns) @@ -276,8 +290,12 @@ private DataTable ReadPipeDelimitedFile(string filePath, List<(string Name, int? { if (DateTypes.Contains(col.DataType)) dataTable.Columns.Add(col.Name, typeof(DateTime)).AllowDBNull = true; + else if (LongTypes.Contains(col.DataType)) + dataTable.Columns.Add(col.Name, typeof(long)).AllowDBNull = true; else if (IntTypes.Contains(col.DataType)) dataTable.Columns.Add(col.Name, typeof(int)).AllowDBNull = true; + else if (BinaryTypes.Contains(col.DataType)) + dataTable.Columns.Add(col.Name, typeof(byte[])).AllowDBNull = true; else dataTable.Columns.Add(col.Name, typeof(string)); } @@ -294,7 +312,7 @@ private DataTable ReadPipeDelimitedFile(string filePath, List<(string Name, int? if (string.IsNullOrWhiteSpace(value)) { - row[i] = DateTypes.Contains(col.DataType) || IntTypes.Contains(col.DataType) + row[i] = DateTypes.Contains(col.DataType) || IntTypes.Contains(col.DataType) || LongTypes.Contains(col.DataType) || BinaryTypes.Contains(col.DataType) ? DBNull.Value : (object)""; continue; @@ -306,10 +324,26 @@ private DataTable ReadPipeDelimitedFile(string filePath, List<(string Name, int? ? dt : DBNull.Value; } + else if (LongTypes.Contains(col.DataType)) + { + row[i] = long.TryParse(value, out var l) ? l : DBNull.Value; + } else if (IntTypes.Contains(col.DataType)) { row[i] = int.TryParse(value, out var n) ? n : DBNull.Value; } + else if (BinaryTypes.Contains(col.DataType)) + { + try + { + var hex = value.StartsWith("0x", StringComparison.OrdinalIgnoreCase) ? value[2..] : value; + row[i] = Convert.FromHexString(hex); + } + catch + { + row[i] = DBNull.Value; + } + } else { var maxLength = col.MaxLength; From 42e72b4a1295d40fdc4392816f471a7cfe13d054 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Thu, 2 Apr 2026 12:51:26 +0100 Subject: [PATCH 44/49] Fix: Add missing END statement to StageDelius stored procedure --- src/Database/DeliusStagingDb/Stored Procedures/StageDelius.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Database/DeliusStagingDb/Stored Procedures/StageDelius.sql b/src/Database/DeliusStagingDb/Stored Procedures/StageDelius.sql index 7053ae8..d5f802b 100644 --- a/src/Database/DeliusStagingDb/Stored Procedures/StageDelius.sql +++ b/src/Database/DeliusStagingDb/Stored Procedures/StageDelius.sql @@ -21,3 +21,4 @@ BEGIN THROW; END CATCH RETURN 0 +END From 631d657cd822ddf0a7c5566cdeb5398b340887e9 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Thu, 9 Apr 2026 14:21:35 +0100 Subject: [PATCH 45/49] Fix: Ensure empty values are input as NULL instead of empty strings --- src/DbInteractions/Services/DbInteractionService.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/DbInteractions/Services/DbInteractionService.cs b/src/DbInteractions/Services/DbInteractionService.cs index 68d071d..886efc7 100644 --- a/src/DbInteractions/Services/DbInteractionService.cs +++ b/src/DbInteractions/Services/DbInteractionService.cs @@ -312,9 +312,7 @@ private DataTable ReadPipeDelimitedFile(string filePath, List<(string Name, int? if (string.IsNullOrWhiteSpace(value)) { - row[i] = DateTypes.Contains(col.DataType) || IntTypes.Contains(col.DataType) || LongTypes.Contains(col.DataType) || BinaryTypes.Contains(col.DataType) - ? DBNull.Value - : (object)""; + row[i] = DBNull.Value; continue; } From 9014174f64bcdcf29c97297f8af9130a1febb1fe Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Wed, 29 Apr 2026 08:40:23 +0100 Subject: [PATCH 46/49] Rebuild: Added new line to trigger rebuild of images to assess Inspector findings --- .github/workflows/build-artifacts.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index a8b2429..6a7d09a 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -33,4 +33,5 @@ jobs: with: name: build-outputs path: ./output/publish - if-no-files-found: error \ No newline at end of file + if-no-files-found: error + \ No newline at end of file From f227e7314041e4d473b56730390e8c7fc085e760 Mon Sep 17 00:00:00 2001 From: Ryan Kearsley Date: Wed, 29 Apr 2026 10:51:12 +0100 Subject: [PATCH 47/49] Update OpenTelemetry package versions to latest releases --- Directory.Packages.props | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index d299cb9..c9e2c9d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -48,11 +48,11 @@ - - - - - + + + + + From 6e4b91bda4a57a1108b39bee148c471a68caa0de Mon Sep 17 00:00:00 2001 From: Sam Gibson Date: Thu, 30 Apr 2026 16:44:30 +0100 Subject: [PATCH 48/49] Revert "Bulk Copy" functionality This caused significant issues with some SQL data types and resulted in imported failures. --- .github/workflows/build-artifacts.yml | 3 +- Directory.Packages.props | 10 +- .../Stored Procedures/StageDelius.sql | 45 ++++- .../Stored Procedures/Import.sql | 62 +++++- .../Services/DbInteractionService.cs | 189 +++--------------- src/FileSync/appsettings.json | 2 +- .../EnvironmentSetup/FileLocations.cs | 5 - 7 files changed, 126 insertions(+), 190 deletions(-) diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index 6a7d09a..a8b2429 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -33,5 +33,4 @@ jobs: with: name: build-outputs path: ./output/publish - if-no-files-found: error - \ No newline at end of file + if-no-files-found: error \ No newline at end of file diff --git a/Directory.Packages.props b/Directory.Packages.props index c9e2c9d..d299cb9 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -48,11 +48,11 @@ - - - - - + + + + + diff --git a/src/Database/DeliusStagingDb/Stored Procedures/StageDelius.sql b/src/Database/DeliusStagingDb/Stored Procedures/StageDelius.sql index d5f802b..df6f73e 100644 --- a/src/Database/DeliusStagingDb/Stored Procedures/StageDelius.sql +++ b/src/Database/DeliusStagingDb/Stored Procedures/StageDelius.sql @@ -1,19 +1,47 @@ CREATE PROCEDURE [DeliusStaging].[StageDelius] - @processedFile VARCHAR(50) + @basePath VARCHAR(100), -- Should be in the format /app/delius/{fileName}. + @processedFile VARCHAR(50), + @inContainer VARCHAR(1) AS -BEGIN - SET NOCOUNT ON; - SET DATEFORMAT DMY; + IF @inContainer = 'N' + BEGIN + SET NOCOUNT ON; + SET DATEFORMAT DMY; + END; - BEGIN TRANSACTION; + CREATE TABLE #TableNames(Id int Primary Key Identity, fileName VARCHAR(50)); + INSERT INTO #TableNames(fileName) VALUES + ('AdditionalIdentifier'), ('AliasDetails'), ('Disability'), ('Disposal'), ('EventDetails'), ('Header'), ('MainOffence'), + ('OAS'), ('OffenderAddress'), ('OffenderManager'), ('OffenderManagerBuildings'), ('OffenderManagerTeam'), + ('OffenderToOffenderManagerMappings'), ('Offenders'), ('OffenderTransfer'), ('PersonalCircumstances'), + ('Provision'), ('RegistrationDetails'), ('Requirement'); + + DECLARE @i int; + DECLARE @sql NVARCHAR(250); + DECLARE @fileName VARCHAR(150); + + SET @i = 1; + + BEGIN TRANSACTION; BEGIN TRY - DECLARE @retMessage VARCHAR(500); - EXEC [DeliusStaging].[StandardiseData] @retMessage OUTPUT; + WHILE @i <= (Select COUNT(*) FROM #TableNames) + BEGIN + SET @fileName = (SELECT fileName FROM #TableNames WHERE Id = @i); + SET @sql = 'BULK INSERT DeliusStaging.' + @fileName + ' FROM ''' + @basePath + @fileName + '.txt''' + + ' WITH (FieldTerminator=''|'', RowTerminator = ''0x0d0a'', MAXERRORS = 1000)'; + + EXEC (@sql); + SET @i = @i+1; + END + + --Standardise Delius Data + Declare @retMessage varchar(500); + EXEC [DeliusStaging].[StandardiseData] @retMessage; UPDATE [DeliusRunningPictureDb].[DeliusRunningPicture].[ProcessedFiles] SET [Status] = 'Imported' WHERE FileName = @processedFile; - + COMMIT TRANSACTION; END TRY BEGIN CATCH @@ -21,4 +49,3 @@ BEGIN THROW; END CATCH RETURN 0 -END diff --git a/src/Database/OfflocStagingDb/Stored Procedures/Import.sql b/src/Database/OfflocStagingDb/Stored Procedures/Import.sql index df79ccc..e73abb1 100644 --- a/src/Database/OfflocStagingDb/Stored Procedures/Import.sql +++ b/src/Database/OfflocStagingDb/Stored Procedures/Import.sql @@ -2,23 +2,75 @@ -- Author: Daniel Fenelon -- Create date: 18/03/2024 -- Description: Import of the OffLoc files --- Note: File loading removed - data is now staged via SqlBulkCopy --- in the DbInteractions service. This procedure handles --- post-load standardisation and status update only. -- ============================================= CREATE PROCEDURE [OfflocStaging].[Import] + @basePath VARCHAR(100), @processedFile VARCHAR(50) AS BEGIN SET NOCOUNT ON; SET DATEFORMAT DMY; + + CREATE TABLE #TableNames + ( + Id INT PRIMARY KEY IDENTITY, + fileName VARCHAR(50) NOT NULL + ); + INSERT INTO #TableNames + ( + fileName + ) + VALUES + ('Activities'), + ('Addresses'), + ('Agencies'), + ('Assessments'), + ('Bookings'), + ('Employment'), + ('Flags'), + ('Identifiers'), + ('IncentiveLevel'), + ('Locations'), + ('MainOffence'), + ('Movements'), + ('OffenderAgencies'), + ('OffenderStatus'), + ('OtherOffences'), + ('PersonalDetails'), + ('PNC'), + ('PreviousPrisonNumbers'), + ('SentenceInformation'), + ('SexOffenders'); + + DECLARE @i INT; + DECLARE @sql VARCHAR(250); + DECLARE @fileName VARCHAR(150); + + SET @i = 1; BEGIN TRANSACTION; BEGIN TRY - DECLARE @retMessage VARCHAR(500); - EXEC [OfflocStaging].[StandardiseData] @retMessage OUTPUT; + WHILE @i <= (SELECT COUNT(*)FROM #TableNames) + BEGIN + SET @fileName = + ( + SELECT fileName FROM #TableNames WHERE Id = @i + ); + SET @sql + = N'BULK INSERT OfflocStaging.' + @fileName + N' FROM ''' + @basePath + @fileName + '.txt''' + + N' WITH (FieldTerminator = ''|'', RowTerminator = ''0x0d0a'', MAXERRORS = 1000)'; + EXEC (@sql); + + SET @i = @i + 1; + END; + DECLARE @stringDate CHAR(8); + SET @stringDate = SUBSTRING(@processedFile, 18, 8); + + --Standardise Offloc Data + Declare @retMessage varchar(500); + EXEC [OfflocStaging].[StandardiseData] @retMessage; UPDATE [OfflocRunningPictureDb].[OfflocRunningPicture].[ProcessedFiles] SET [Status] = 'Imported' diff --git a/src/DbInteractions/Services/DbInteractionService.cs b/src/DbInteractions/Services/DbInteractionService.cs index 886efc7..6416fdd 100644 --- a/src/DbInteractions/Services/DbInteractionService.cs +++ b/src/DbInteractions/Services/DbInteractionService.cs @@ -5,7 +5,6 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Options; using System.Data; -using System.Globalization; namespace DbInteractions.Services; @@ -14,6 +13,7 @@ public class DbInteractionService : IDbInteractionService private readonly IFileLocations fileLocations; private readonly ServerConfiguration serverConfig; private readonly IConfiguration configuration; + private readonly bool inContainer; private readonly IMessageService messageService; public DbInteractionService( @@ -26,6 +26,7 @@ public DbInteractionService( this.configuration = config; this.fileLocations = fileLocations; this.messageService = messageService; + this.inContainer = config.GetValue("RUNNING_IN_CONTAINER"); } public async Task GetProcessedDeliusFileNames() @@ -149,48 +150,34 @@ SELECT FileName as [Name] FROM [OfflocRunningPicture].[ProcessedFiles] public async Task StageDelius(string fileName, string filePath) { string folderName = filePath.Split('/').Last(); - string folderPath = Path.Combine(fileLocations.deliusOutput, folderName); await messageService.PublishAsync(new StatusUpdateMessage($"Delius staging started for file number {fileName}")); - var tableNames = new[] + string containerFlag = string.Empty; + if (inContainer) { - "AdditionalIdentifier", "AliasDetails", "Disability", "Disposal", "EventDetails", - "Header", "MainOffence", "OAS", "OffenderAddress", "OffenderManager", - "OffenderManagerBuildings", "OffenderManagerTeam", "OffenderToOffenderManagerMappings", - "Offenders", "OffenderTransfer", "PersonalCircumstances", "Provision", - "RegistrationDetails", "Requirement" - }; + containerFlag = "Y"; + } + else + { + containerFlag = "N"; + } var deliusConn = new SqlConnection(configuration.GetConnectionString("DeliusStagingDb")!); using (deliusConn) { await deliusConn.OpenAsync(); + SqlCommand cmd = new SqlCommand(serverConfig.DeliusStagingProcedure, deliusConn); + cmd.CommandType = CommandType.StoredProcedure; + cmd.CommandTimeout = 600; + cmd.Parameters.AddWithValue("@basePath", $"{fileLocations.deliusOutput}/{folderName}/"); + cmd.Parameters.AddWithValue("@processedFile", fileName); + cmd.Parameters.AddWithValue("@inContainer", containerFlag); + try { - foreach (var table in tableNames) - { - var tableFilePath = Path.Combine(folderPath, $"{table}.txt"); - if (!File.Exists(tableFilePath)) continue; - - var columns = await GetTableColumns(deliusConn, "DeliusStaging", table); - var dataTable = ReadPipeDelimitedFile(tableFilePath, columns); - - using var bulkCopy = new SqlBulkCopy(deliusConn) - { - DestinationTableName = $"[DeliusStaging].[{table}]", - BulkCopyTimeout = 600 - }; - await bulkCopy.WriteToServerAsync(dataTable); - } - - // Calls StandardiseData + updates ProcessedFiles - SqlCommand command = new SqlCommand(serverConfig.DeliusStagingProcedure, deliusConn); - command.CommandType = CommandType.StoredProcedure; - command.CommandTimeout = 600; - command.Parameters.AddWithValue("@processedFile", fileName); - await command.ExecuteNonQueryAsync(); + var res = await cmd.ExecuteNonQueryAsync(); } catch (Exception e) { @@ -199,54 +186,27 @@ public async Task StageDelius(string fileName, string filePath) } } - await messageService.PublishAsync(new StatusUpdateMessage($"Delius Staging completed.")); + await messageService.PublishAsync(new StatusUpdateMessage($"Delius staging finished for file {fileName}.")); } public async Task StageOffloc(string fileName) { string folderName = fileName.Split('.').First(); - string folderPath = Path.Combine(fileLocations.offlocOutput, folderName); - await messageService.PublishAsync(new StatusUpdateMessage($"Offloc staging started for file {fileName}.")); - var tableNames = new[] - { - "Activities", "Addresses", "Agencies", "Assessments", "Bookings", - "Employment", "Flags", "Identifiers", "IncentiveLevel", "Locations", - "MainOffence", "Movements", "OffenderAgencies", "OffenderStatus", - "OtherOffences", "PersonalDetails", "PNC", "PreviousPrisonNumbers", - "SentenceInformation", "SexOffenders" - }; - var offlocConn = new SqlConnection(configuration.GetConnectionString("OfflocStagingDb")!); using (offlocConn) { await offlocConn.OpenAsync(); + SqlCommand command = new SqlCommand(serverConfig.OfflocStagingProcedure, offlocConn); + command.CommandTimeout = 600; + command.CommandType = CommandType.StoredProcedure; + command.Parameters.AddWithValue("@basePath", $"{fileLocations.offlocOutput}/{folderName}/"); + command.Parameters.AddWithValue("@processedFile", fileName); try { - foreach (var table in tableNames) - { - var filePath = Path.Combine(folderPath, $"{table}.txt"); - if (!File.Exists(filePath)) continue; - - var columns = await GetTableColumns(offlocConn, "OfflocStaging", table); - var dataTable = ReadPipeDelimitedFile(filePath, columns); - - using var bulkCopy = new SqlBulkCopy(offlocConn) - { - DestinationTableName = $"[OfflocStaging].[{table}]", - BulkCopyTimeout = 600 - }; - await bulkCopy.WriteToServerAsync(dataTable); - } - - // Calls StandardiseData + updates ProcessedFiles - SqlCommand command = new SqlCommand(serverConfig.OfflocStagingProcedure, offlocConn); - command.CommandTimeout = 600; - command.CommandType = CommandType.StoredProcedure; - command.Parameters.AddWithValue("@processedFile", fileName); - await command.ExecuteNonQueryAsync(); + var result = await command.ExecuteNonQueryAsync(); } catch (Exception e) { @@ -255,104 +215,7 @@ public async Task StageOffloc(string fileName) } } - await messageService.PublishAsync(new StatusUpdateMessage($"Offloc Staging completed.")); - } - - private async Task> GetTableColumns(SqlConnection conn, string schema, string table) - { - var columns = new List<(string Name, int? MaxLength, string DataType)>(); - using var cmd = new SqlCommand( - "SELECT COLUMN_NAME, CHARACTER_MAXIMUM_LENGTH, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = @schema AND TABLE_NAME = @table ORDER BY ORDINAL_POSITION", - conn); - cmd.Parameters.AddWithValue("@schema", schema); - cmd.Parameters.AddWithValue("@table", table); - using var reader = await cmd.ExecuteReaderAsync(); - while (await reader.ReadAsync()) - { - var name = reader.GetString(0); - var maxLength = reader.IsDBNull(1) ? (int?)null : reader.GetInt32(1); - var dataType = reader.GetString(2); - columns.Add((name, maxLength, dataType)); - } - return columns; - } - - private static readonly string[] DateTypes = ["date", "datetime", "datetime2", "smalldatetime"]; - private static readonly string[] IntTypes = ["int", "smallint", "tinyint"]; - private static readonly string[] LongTypes = ["bigint"]; - private static readonly string[] BinaryTypes = ["varbinary", "binary"]; - private static readonly CultureInfo UkCulture = new CultureInfo("en-GB"); - - private DataTable ReadPipeDelimitedFile(string filePath, List<(string Name, int? MaxLength, string DataType)> columns) - { - var dataTable = new DataTable(); - foreach (var col in columns) - { - if (DateTypes.Contains(col.DataType)) - dataTable.Columns.Add(col.Name, typeof(DateTime)).AllowDBNull = true; - else if (LongTypes.Contains(col.DataType)) - dataTable.Columns.Add(col.Name, typeof(long)).AllowDBNull = true; - else if (IntTypes.Contains(col.DataType)) - dataTable.Columns.Add(col.Name, typeof(int)).AllowDBNull = true; - else if (BinaryTypes.Contains(col.DataType)) - dataTable.Columns.Add(col.Name, typeof(byte[])).AllowDBNull = true; - else - dataTable.Columns.Add(col.Name, typeof(string)); - } - - foreach (var line in File.ReadLines(filePath)) - { - if (string.IsNullOrWhiteSpace(line)) continue; - var values = line.Split('|'); - var row = dataTable.NewRow(); - for (int i = 0; i < Math.Min(values.Length, columns.Count); i++) - { - var value = values[i]; - var col = columns[i]; - - if (string.IsNullOrWhiteSpace(value)) - { - row[i] = DBNull.Value; - continue; - } - - if (DateTypes.Contains(col.DataType)) - { - row[i] = DateTime.TryParse(value, UkCulture, DateTimeStyles.None, out var dt) - ? dt - : DBNull.Value; - } - else if (LongTypes.Contains(col.DataType)) - { - row[i] = long.TryParse(value, out var l) ? l : DBNull.Value; - } - else if (IntTypes.Contains(col.DataType)) - { - row[i] = int.TryParse(value, out var n) ? n : DBNull.Value; - } - else if (BinaryTypes.Contains(col.DataType)) - { - try - { - var hex = value.StartsWith("0x", StringComparison.OrdinalIgnoreCase) ? value[2..] : value; - row[i] = Convert.FromHexString(hex); - } - catch - { - row[i] = DBNull.Value; - } - } - else - { - var maxLength = col.MaxLength; - if (maxLength.HasValue && maxLength.Value > 0 && value.Length > maxLength.Value) - value = value[..maxLength.Value]; - row[i] = value; - } - } - dataTable.Rows.Add(row); - } - return dataTable; + await messageService.PublishAsync(new StatusUpdateMessage($"Offloc staging finished for file {fileName}.")); } //Calls merge and then on completion public async Task StandardiseDeliusStaging() diff --git a/src/FileSync/appsettings.json b/src/FileSync/appsettings.json index 879dcf3..4cdb7bd 100644 --- a/src/FileSync/appsettings.json +++ b/src/FileSync/appsettings.json @@ -23,7 +23,7 @@ "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] }, "SyncOptions": { - "ProcessOnStartup": true, + "ProcessOnStartup": false, "ProcessOnCompletion": true, "ProcessOnTimer": true, "ProcessTimerIntervalSeconds": 900, diff --git a/src/Libraries/EnvironmentSetup/FileLocations.cs b/src/Libraries/EnvironmentSetup/FileLocations.cs index e85ff1e..6ff2225 100644 --- a/src/Libraries/EnvironmentSetup/FileLocations.cs +++ b/src/Libraries/EnvironmentSetup/FileLocations.cs @@ -18,11 +18,6 @@ public class FileLocations : IFileLocations public FileLocations(string basePath) { this._basePath = basePath.Replace("~", Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)); - - Directory.CreateDirectory(deliusInput); - Directory.CreateDirectory(deliusOutput); - Directory.CreateDirectory(offlocInput); - Directory.CreateDirectory(offlocOutput); } public string basePath { get => _basePath; } From 29fc1a5b201c1e3bf3af8aff5463aac2a26abaf5 Mon Sep 17 00:00:00 2001 From: Sam Gibson Date: Fri, 1 May 2026 11:10:24 +0100 Subject: [PATCH 49/49] Add a separate base path configuration DbInteractions --- .github/workflows/deploy-reusable.yml | 1 + infra/services.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-reusable.yml b/.github/workflows/deploy-reusable.yml index 3192b42..09353c6 100644 --- a/.github/workflows/deploy-reusable.yml +++ b/.github/workflows/deploy-reusable.yml @@ -158,6 +158,7 @@ jobs: ["CatsRabbitMQ"]="ConnectionStrings__CatsRabbitMQ" ["RabbitMQ"]="ConnectionStrings__RabbitMQ" ["DMSFilesBasePath"]="DMSFilesBasePath" + ["DMSFilesBasePath_DbInteractions"]="DMSFilesBasePath" ["Sentry_Dsn"]="Sentry_Dsn" ) diff --git a/infra/services.json b/infra/services.json index 458ae1c..05b47b2 100644 --- a/infra/services.json +++ b/infra/services.json @@ -101,7 +101,7 @@ "taskDef": "infra/task-definitions/dbinteractions-task-def.json", "container": "dbinteractions-container", "secrets": [ - "DMSFilesBasePath", + "DMSFilesBasePath_DbInteractions", "DeliusRunningPictureDb", "OfflocRunningPictureDb", "DeliusStagingDb",