Merge SQS story into main

.gitattributes

@@ -0,0 +1 @@
+*.sh text eol=lf

.github/workflows/check-pull-request.yml

@@ -19,41 +19,32 @@ jobs:
     steps:
       - uses: actions/setup-dotnet@v3
         with:
-          dotnet-version: |
-            8.0
-
+          dotnet-version: '8.0.x'    
       - name: Check out code
         uses: actions/checkout@v3
         with:
-          path: working-dir
-
-      - name: Verify Format
+          path: working-dir    
+      - name: Restore
+        run: dotnet restore ./working-dir/KeeperData.Api.sln  
+      - name: Build
+        run: dotnet build ./working-dir/KeeperData.Api.sln --configuration Release --no-restore -warnaserror  
+      - name: Verify format
         run: dotnet format ./working-dir/KeeperData.Api.sln --verify-no-changes --verbosity diagnostic
-
-      - name: Run docker-compose
-        uses: hoverkraft-tech/compose-action@v2.0.1
-        with:
-          compose-file: "./working-dir/docker-compose.yml"
-
-      - name: Make Script Executable
-        run: chmod +x ./working-dir/wait-for-docker-logs.sh
-
-      - name: Wait for Localstack
-        run: ./working-dir/wait-for-docker-logs.sh localstack-emulator "Ready." 60
-
-      - name: Test
-        run: dotnet test ./working-dir/KeeperData.Api.sln --logger "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true" --collect:"XPlat Code Coverage" --results-directory ./coverage
-
-      - name: install dotnet coverage
-        run: dotnet tool install --global dotnet-coverage        
-
-      - name: merge coverage reports
-        run:  dotnet-coverage merge *.cobertura.xml --recursive --output merged.cobertura.xml --output-format cobertura
-
-      - name: Code Coverage Report
+      - name: Test (unit and component)
+        run: dotnet test ./working-dir/KeeperData.Api.sln --filter Dependence!=localstack --logger "GitHubActions;summary.includePassedTests=true;summary.includeSkippedTests=true" --collect:"XPlat Code Coverage" --results-directory ./coverage
+      - name: Install ReportGenerator
+        run: dotnet tool install --global dotnet-reportgenerator-globaltool
+      - name: Generate Coverage Summary
+        run: |
+          reportgenerator \
+            -reports:"./coverage/**/coverage.cobertura.xml" \
+            -targetdir:"coverage-report" \
+            -reporttypes:"Cobertura;MarkdownSummary" \
+            -assemblyfilters:+KeeperData.*
+      - name: Code coverage report
         uses: irongut/CodeCoverageSummary@v1.3.0
         with:
-          filename: merged.cobertura.xml
+          filename: coverage-report/Cobertura.xml
           badge: true
           fail_below_min: false
           format: markdown
@@ -62,14 +53,41 @@ jobs:
           indicators: true
           output: both
           thresholds: '60 80'
-
       - name: Add Coverage PR Comment
         uses: marocchino/sticky-pull-request-comment@v2
         if: github.event_name == 'pull_request'
         with:
           recreate: true
           path: code-coverage-results.md
 
+  integration-tests:
+    name: Run Integration Tests
+    runs-on: ubuntu-latest
+    needs: pr-validator
+    steps:
+      - uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: '8.0.x'
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          path: working-dir
+      - name: Make LocalStack init script executable
+        run: chmod +x ./working-dir/compose/start-localstack.sh
+      - name: Run docker-compose
+        uses: hoverkraft-tech/compose-action@v2.0.1
+        with:
+          compose-file: |
+            ./working-dir/docker-compose.yml
+            ./working-dir/docker-compose.override.yml
+      - name: Make script executable
+        run: chmod +x ./working-dir/wait-for-docker-logs.sh
+      - name: Wait for localstack
+        run: ./working-dir/wait-for-docker-logs.sh localstack-emulator "Bootstrapping Complete" 60
+      - name: Wait for keeperdata_api to be ready
+        run: ./working-dir/wait-for-docker-logs.sh keeperdata_api "Now listening on" 60
+      - name: Run integration tests
+        run: dotnet test ./working-dir/KeeperData.Api.sln --filter Dependence=localstack
 
     ## SonarCloud
     ## Uncomment to unable SonarCloud scan

.github/workflows/publish.yml

@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - main
+      - develop
+      - 'feature/**'
 
 permissions:
   id-token: write

.gitignore

@@ -267,3 +267,6 @@ __pycache__/
 
 library.db
 .envrc
+/localstack_data/cache
+/coverage
+/coverage-report

Dockerfile

@@ -1,10 +1,8 @@
-# Base dotnet image
+# Base dotnet image
 FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base
 WORKDIR /app
 EXPOSE 80
 EXPOSE 443
-EXPOSE 8080
-EXPOSE 8081
 
 # Add curl to template.
 # CDP PLATFORM HEALTHCHECK REQUIREMENT
@@ -16,19 +14,26 @@ RUN apt update && \
 # Build stage image
 FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
 ARG BUILD_CONFIGURATION=Release
+ENV BUILD_CONFIGURATION=${BUILD_CONFIGURATION}
 WORKDIR /src
-COPY . .
 
-# unit test and code coverage
-RUN dotnet test KeeperData.Api.sln --filter Dependence!=localstack
-RUN dotnet restore "KeeperData.Api.sln"
-RUN dotnet build "src/KeeperData.Api/KeeperData.Api.csproj" -c $BUILD_CONFIGURATION -o /app/build /p:UseAppHost=false
+COPY ["src/KeeperData.Api/KeeperData.Api.csproj", "KeeperData.Api/"]
+COPY ["src/KeeperData.Infrastructure/KeeperData.Infrastructure.csproj", "KeeperData.Infrastructure/"]
+COPY ["src/KeeperData.Application/KeeperData.Application.csproj", "KeeperData.Application/"]
+COPY ["src/KeeperData.Core/KeeperData.Core.csproj", "KeeperData.Core/"]
+
+RUN dotnet restore "KeeperData.Api/KeeperData.Api.csproj" -r linux-x64 -v n
+RUN dotnet restore "KeeperData.Infrastructure/KeeperData.Infrastructure.csproj" -r linux-x64 -v n
+RUN dotnet restore "KeeperData.Application/KeeperData.Application.csproj" -r linux-x64 -v n
+RUN dotnet restore "KeeperData.Core/KeeperData.Core.csproj" -r linux-x64 -v n
+
+COPY ["src/", "."]
 
 FROM build AS publish
-ARG BUILD_CONFIGURATION=Release
-RUN dotnet publish "src/KeeperData.Api" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false
-ENV ASPNETCORE_FORWARDEDHEADERS_ENABLED=true
+WORKDIR "/src/KeeperData.Api"
+RUN dotnet publish "KeeperData.Api.csproj" -v n -c ${BUILD_CONFIGURATION} -o /app/publish -r linux-x64 --no-restore /p:UseAppHost=false
 
+ENV ASPNETCORE_FORWARDEDHEADERS_ENABLED=true
 
 # Final production image
 FROM base AS final

KeeperData.Api.sln

@@ -13,24 +13,23 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{9780B59B
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Infrastructure", "src\KeeperData.Infrastructure\KeeperData.Infrastructure.csproj", "{F07A8DFD-D7A7-42C8-9850-F0C336DEC045}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Data", "src\KeeperData.Data\KeeperData.Data.csproj", "{8256795B-5937-43DF-95C2-D41053D2D27E}"
-EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Common", "src\KeeperData.Common\KeeperData.Common.csproj", "{F4DD4797-B666-4175-83FC-0E57F8C3DB80}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Api.Tests.Integration", "tests\KeeperData.Api.Tests.Integration\KeeperData.Api.Tests.Integration.csproj", "{72D99550-9D4F-4374-9344-BC018111906D}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "solutionitems", "solutionitems", "{B0A5300E-181B-4F81-9C8C-64F69DAAC024}"
 	ProjectSection(SolutionItems) = preProject
 		.gitignore = .gitignore
-		docker-compose.yml = docker-compose.yml
 		Dockerfile = Dockerfile
 	EndProjectSection
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Core.Tests.Unit", "tests\KeeperData.Core.Tests.Unit\KeeperData.Core.Tests.Unit.csproj", "{A1C6A0AD-6C5D-4698-8047-FFFB9ECD9FD9}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Infrastructure.Tests.Unit", "tests\KeeperData.Infrastructure.Tests.Unit\KeeperData.Infrastructure.Tests.Unit.csproj", "{5BFBB1F5-7D75-411B-9A71-1325C1E22E09}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Common.Tests.Unit", "tests\KeeperData.Common.Tests.Unit\KeeperData.Common.Tests.Unit.csproj", "{F16C919A-8FB1-4AC9-B45F-C2E29F62FF7A}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Application", "src\KeeperData.Application\KeeperData.Application.csproj", "{C628C38C-B7BF-4CBB-9A44-F0ED36F3B48F}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KeeperData.Application.Tests.Unit", "tests\KeeperData.Application.Tests.Unit\KeeperData.Application.Tests.Unit.csproj", "{BA4E8D8A-FFCE-4ECD-8C55-90194019C765}"
+EndProject
+Project("{E53339B2-1760-4266-BCC7-CA923CBCF16C}") = "docker-compose", "docker-compose.dcproj", "{81DDED9D-158B-E303-5F62-77A2896D2A5A}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -54,14 +53,6 @@ Global
 		{F07A8DFD-D7A7-42C8-9850-F0C336DEC045}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{F07A8DFD-D7A7-42C8-9850-F0C336DEC045}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{F07A8DFD-D7A7-42C8-9850-F0C336DEC045}.Release|Any CPU.Build.0 = Release|Any CPU
-		{8256795B-5937-43DF-95C2-D41053D2D27E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{8256795B-5937-43DF-95C2-D41053D2D27E}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{8256795B-5937-43DF-95C2-D41053D2D27E}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{8256795B-5937-43DF-95C2-D41053D2D27E}.Release|Any CPU.Build.0 = Release|Any CPU
-		{F4DD4797-B666-4175-83FC-0E57F8C3DB80}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F4DD4797-B666-4175-83FC-0E57F8C3DB80}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F4DD4797-B666-4175-83FC-0E57F8C3DB80}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F4DD4797-B666-4175-83FC-0E57F8C3DB80}.Release|Any CPU.Build.0 = Release|Any CPU
 		{72D99550-9D4F-4374-9344-BC018111906D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{72D99550-9D4F-4374-9344-BC018111906D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{72D99550-9D4F-4374-9344-BC018111906D}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -74,10 +65,14 @@ Global
 		{5BFBB1F5-7D75-411B-9A71-1325C1E22E09}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{5BFBB1F5-7D75-411B-9A71-1325C1E22E09}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{5BFBB1F5-7D75-411B-9A71-1325C1E22E09}.Release|Any CPU.Build.0 = Release|Any CPU
-		{F16C919A-8FB1-4AC9-B45F-C2E29F62FF7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F16C919A-8FB1-4AC9-B45F-C2E29F62FF7A}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F16C919A-8FB1-4AC9-B45F-C2E29F62FF7A}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F16C919A-8FB1-4AC9-B45F-C2E29F62FF7A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C628C38C-B7BF-4CBB-9A44-F0ED36F3B48F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C628C38C-B7BF-4CBB-9A44-F0ED36F3B48F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C628C38C-B7BF-4CBB-9A44-F0ED36F3B48F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C628C38C-B7BF-4CBB-9A44-F0ED36F3B48F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BA4E8D8A-FFCE-4ECD-8C55-90194019C765}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BA4E8D8A-FFCE-4ECD-8C55-90194019C765}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BA4E8D8A-FFCE-4ECD-8C55-90194019C765}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BA4E8D8A-FFCE-4ECD-8C55-90194019C765}.Release|Any CPU.Build.0 = Release|Any CPU
 		{81DDED9D-158B-E303-5F62-77A2896D2A5A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{81DDED9D-158B-E303-5F62-77A2896D2A5A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{81DDED9D-158B-E303-5F62-77A2896D2A5A}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -91,7 +86,7 @@ Global
 		{72D99550-9D4F-4374-9344-BC018111906D} = {9780B59B-FFF7-481C-AA15-072631C6736C}
 		{A1C6A0AD-6C5D-4698-8047-FFFB9ECD9FD9} = {9780B59B-FFF7-481C-AA15-072631C6736C}
 		{5BFBB1F5-7D75-411B-9A71-1325C1E22E09} = {9780B59B-FFF7-481C-AA15-072631C6736C}
-		{F16C919A-8FB1-4AC9-B45F-C2E29F62FF7A} = {9780B59B-FFF7-481C-AA15-072631C6736C}
+		{BA4E8D8A-FFCE-4ECD-8C55-90194019C765} = {9780B59B-FFF7-481C-AA15-072631C6736C}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {23761809-E330-42D6-B303-C5568B1FB035}

compose/aws.env

@@ -2,3 +2,4 @@ AWS_REGION=eu-west-2
 AWS_DEFAULT_REGION=eu-west-2
 AWS_ACCESS_KEY_ID=test
 AWS_SECRET_ACCESS_KEY=test
+COMPOSE_PROJECT_NAME=keeperdata_api

compose/start-localstack.sh

@@ -1,11 +1,93 @@
 #!/bin/bash
+sed -i 's/\r$//' "$0"
+
 export AWS_REGION=eu-west-2
 export AWS_DEFAULT_REGION=eu-west-2
 export AWS_ACCESS_KEY_ID=test
 export AWS_SECRET_ACCESS_KEY=test
 
+set -e
+
 # S3 buckets
-# aws --endpoint-url=http://localhost:4566 s3 mb s3://my-bucket
+echo "Bootstrapping S3 setup..."
+
+## Create 'test-comparison-reports-bucket' Bucket
+existing_bucket=$(awslocal s3api list-buckets \
+  --query "Buckets[?Name=='test-comparison-reports-bucket'].Name" \
+  --output text)
+
+if [ "$existing_bucket" == "test-comparison-reports-bucket" ]; then
+  echo "S3 bucket already exists: test-comparison-reports-bucket"
+else
+  awslocal s3api create-bucket --bucket test-comparison-reports-bucket --region eu-west-2 \
+    --create-bucket-configuration LocationConstraint=eu-west-2 \
+    --endpoint-url=http://localhost:4566
+  echo "S3 bucket created: test-comparison-reports-bucket"
+fi
+
+echo "Bootstrapping SQS setup..."
+
+# Create SQS resources
+queue_url=$(awslocal sqs create-queue  \
+  --queue-name ls_keeper_data_intake_queue \
+  --endpoint-url=http://localhost:4566 \
+  --output text \
+  --query 'QueueUrl')
+
+echo "SQS Queue created: $queue_url"
+
+# Get the SQS Queue ARN
+queue_arn=$(awslocal sqs get-queue-attributes \
+  --queue-url "$queue_url" \
+  --attribute-name QueueArn \
+  --output text \
+  --query 'Attributes.QueueArn')
+
+echo "SQS Queue ARN: $queue_arn"
+
+# Create SNS Topics
+topic_arn=$(awslocal sns create-topic \
+  --name ls-keeper-data-bridge-events \
+  --endpoint-url=http://localhost:4566 \
+  --output text \
+  --query 'TopicArn')
+
+echo "SNS Topic created: $topic_arn"
+
+# Construct the policy JSON inline with escaped quotes
+policy_json=$(cat <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Principal": "*",
+      "Action": "sqs:SendMessage",
+      "Resource": "$queue_arn",
+      "Condition": {
+        "ArnEquals": {
+          "aws:SourceArn": "$topic_arn"
+        }
+      }
+    }
+  ]
+}
+EOF
+)
+
+# Set SQS policy
+awslocal sqs set-queue-attributes \
+  --queue-url "$queue_url" \
+  --attributes "{\"Policy\": \"$(
+    echo "$policy_json" | jq -c
+  )\"}"
+
+# Subscribe the Queue to the Topic
+awslocal sns subscribe \
+  --topic-arn "$topic_arn" \
+  --protocol sqs \
+  --notification-endpoint "$queue_arn"
+
+echo "SNS Topic subscription complete"
 
-# SQS queues
-# aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name my-queue
+echo "Bootstrapping Complete"

docker-compose.dcproj

@@ -8,6 +8,7 @@
     <DockerLaunchAction>LaunchBrowser</DockerLaunchAction>
     <DockerServiceUrl>{Scheme}://localhost:{ServicePort}/health</DockerServiceUrl>
     <DockerServiceName>keeperdata_api</DockerServiceName>
+    <AdditionalComposeFilePaths>docker-compose.override.yml</AdditionalComposeFilePaths>
   </PropertyGroup>
   <ItemGroup>
     <None Include="docker-compose.override.yml">
@@ -16,4 +17,9 @@
     <None Include="docker-compose.yml" />
     <None Include=".dockerignore" />
   </ItemGroup>
+  <ItemGroup>
+    <DockerComposeServiceReference Include="keeperdata_api">
+      <ProjectPath>src\KeeperData.Api\KeeperData.Api.csproj</ProjectPath>
+    </DockerComposeServiceReference>
+  </ItemGroup>
 </Project>

docker-compose.override.yml

@@ -1,10 +1,14 @@
 services:
   keeperdata_api:
     labels:
-      com.microsoft.visual-studio.project-name: "keeperdata.api"
+      com.microsoft.visual-studio.project-name: "KeeperData.Api"
     environment:
       - ASPNETCORE_ENVIRONMENT=Development
       - ASPNETCORE_HTTP_PORTS=8080
+      - AWS__ServiceURL=http://localstack-emulator:4566
+      - Mongo__DatabaseUri=mongodb://mongodb:27017
+      - StorageConfiguration__ComparisonReportsStorage__BucketName=test-comparison-reports-bucket
+      - QueueConsumerOptions__IntakeEventQueueOptions__QueueUrl=http://sqs.eu-west-2.127.0.0.1:4566/000000000000/ls_keeper_data_intake_queue
     ports:
       - "8080"
     volumes: