Merge pull request #2056 from release/2026.05.1

Release 2026.05.1

Author: divbzero

.github/workflows/_deploy.yaml

@@ -134,7 +134,7 @@ jobs:
       - name: configure aws credentials
         uses: aws-actions/configure-aws-credentials@v6
         with:
-          role-to-assume: arn:aws:iam::917902836630:role/github-oidc-ssm-version20260317121845461900000008
+          role-to-assume: arn:aws:iam::917902836630:role/github-oidc-ssm-version
           role-session-name: OIDC-GHA-session-version
           aws-region: us-east-1
       - name: Store version in SSM

.github/workflows/_setup.yaml

@@ -35,22 +35,22 @@ jobs:
           ENV_NAME="${{ inputs.env-name }}"
           case "$ENV_NAME" in
             "dev")
-              echo 'role=arn:aws:iam::017925157769:role/github-oidc-deploy20260311132710794900000001' >> "$GITHUB_OUTPUT"
+              echo 'role=arn:aws:iam::017925157769:role/github-oidc-deploy' >> "$GITHUB_OUTPUT"
             ;;
             "test")
               echo 'role=arn:aws:iam::641513112151:role/cmiml-test-oidc-github-role' >> "$GITHUB_OUTPUT"
             ;;
             "uat")
-              echo 'role=arn:aws:iam::641513112151:role/github-oidc-deploy20260312130551192100000004' >> "$GITHUB_OUTPUT"
+              echo 'role=arn:aws:iam::641513112151:role/github-oidc-deploy' >> "$GITHUB_OUTPUT"
             ;;
             "stage")
               echo 'role=arn:aws:iam::641513112151:role/cmiml-stage-oidc-github-role' >> "$GITHUB_OUTPUT"
             ;;
             "prod")
-              echo 'role=arn:aws:iam::410431445687:role/github-oidc-deploy20260312143902067900000001' >> "$GITHUB_OUTPUT"
+              echo 'role=arn:aws:iam::410431445687:role/github-oidc-deploy' >> "$GITHUB_OUTPUT"
             ;;
             "prod-dr-us-west-2")
-              echo 'role=arn:aws:iam::973422231492:role/github-oidc-deploy20260313122244970500000002' >> "$GITHUB_OUTPUT"
+              echo 'role=arn:aws:iam::973422231492:role/github-oidc-deploy' >> "$GITHUB_OUTPUT"
             ;;
             *)
               echo "Bad environment name"
@@ -100,10 +100,10 @@ jobs:
           ENV_NAME="${{ inputs.env-name }}"
           case "$ENV_NAME" in
             "prod-dr-us-west-2")
-              echo "role=arn:aws:iam::973422231492:role/github-oidc-ecr20260317131325572800000002" >> "$GITHUB_OUTPUT"
+              echo "role=arn:aws:iam::973422231492:role/github-oidc-ecr-push" >> "$GITHUB_OUTPUT"
               ;;
             *)
-              echo "role=arn:aws:iam::917902836630:role/github-oidc-ecr-push20260317121845560500000009" >> "$GITHUB_OUTPUT"
+              echo "role=arn:aws:iam::917902836630:role/github-oidc-ecr-push" >> "$GITHUB_OUTPUT"
               ;;
           esac
 

pyproject.toml

@@ -16,7 +16,7 @@ dependencies = [
   "azure-storage-blob==12.25.*",
   "bcrypt~=4.3.0",
   "boto3~=1.40.60",
-  "cryptography>=46.0.6",
+  "cryptography>=46.0.7",
   "ddtrace~=3.17.1",
   "fastapi==0.124.*",
   "fastapi-mail>=1.5.3",
@@ -28,7 +28,7 @@ dependencies = [
   "pyjwt>=2.12.0",
   "pymongo==4.13.0",
   "pyotp~=2.9.0",
-  "python-multipart>=0.0.22",
+  "python-multipart>=0.0.26",
   "python-slugify==8.0.4",
   "redis==5.2.*",
   "sentry-sdk~=2.13",
@@ -85,8 +85,8 @@ dev = [
   "polyfactory==2.21.0",
   "pre-commit>=4.5.1",
   "pyld==2.0.4",
-  "pytest==8.4.0",
-  "pytest-asyncio==1.1.0",
+  "pytest>=9.0.3",
+  "pytest-asyncio>=1.3.0",
   "pytest-cov==6.1.1",
   "pytest-env==1.1.5",
   "pytest-httpx",

src/broker.py

@@ -11,7 +11,13 @@
 from infrastructure.logger import logger
 
 broker: AsyncBroker = (
-    AioPikaBroker(settings.rabbitmq.url)
+    AioPikaBroker(
+        settings.rabbitmq.url,
+        exchange_name="curious",
+        queue_name="curious",
+        declare_exchange_kwargs={"durable": True},
+        declare_queues_kwargs={"durable": True},
+    )
     .with_result_backend(RedisAsyncResultBackend(settings.redis.url))
     .with_formatter(JSONFormatter())
 )

src/config/cdn.py

@@ -19,6 +19,16 @@ class CDNSettings(BaseModel):
     secret_key: str | None = None
     access_key: str | None = None
 
+    # KMS
+    bucket_kms_enabled: bool = False
+    bucket_kms_key_id: str | None = None
+
+    bucket_answer_kms_enabled: bool = False
+    bucket_answer_kms_key_id: str | None = None
+
+    bucket_operation_kms_enabled: bool = False
+    bucket_operation_kms_key_id: str | None = None
+
     ## DR settings
     # Override the media bucket name for the DR site
     bucket_override: str | None = None
@@ -35,7 +45,7 @@ class CDNSettings(BaseModel):
     gcp_endpoint_url: str = "https://storage.googleapis.com"
 
     # Custom Object store endpoint URL
-    # Usually a custom S3 endpoint or GCP, etc.
+    # Usually this is a custom S3 endpoint or GCP, etc.
     # Locally for minio type stores it is in the form http://localhost:9000
     endpoint_url: str | None = None
 
@@ -51,6 +61,16 @@ def validate_settings(self) -> Self:
         """Validate that domain or endpoint is set.  Cannot be both"""
         if self.domain and (self.endpoint_url or self.storage_address):
             raise ValueError("Either domain or endpoint_url must be set, not both.")
+
+        if self.bucket_kms_enabled and not self.bucket_kms_key_id:
+            raise ValueError("bucket_kms_key_id must be set if bucket_kms_enabled is True")
+
+        if self.bucket_answer_kms_enabled and not self.bucket_answer_kms_key_id:
+            raise ValueError("bucket_answer_kms_key_id must be set if bucket_answer_kms_enabled is True")
+
+        if self.bucket_operation_kms_enabled and not self.bucket_operation_kms_key_id:
+            raise ValueError("bucket_operation_kms_key_id must be set if bucket_operation_kms_enabled is True")
+
         return self
 
     @property

src/infrastructure/storage/storage_client.py

@@ -7,7 +7,7 @@
 from typing import Any, BinaryIO
 
 import boto3
-import httpx
+import requests
 from botocore.config import Config
 from botocore.exceptions import ClientError, EndpointConnectionError
 from ddtrace.trace import tracer
@@ -176,7 +176,21 @@ async def list_object(self, key: str):
 
     def generate_presigned_post(self, key) -> dict[str, Any]:
         # Not needed ThreadPoolExecutor because there is no any IO operation (no API calls to s3)
-        return self.client.generate_presigned_post(self._get_bucket_name(), key, ExpiresIn=self.config.ttl_signed_urls)
+        fields = {}
+        conditions = []
+        if self.config.kms_enabled:
+            fields = {
+                "x-amz-server-side-encryption": "aws:kms",
+                "x-amz-server-side-encryption-aws-kms-key-id": self.config.kms_key_id,
+            }
+            conditions = [
+                {"x-amz-server-side-encryption": "aws:kms"},
+                {"x-amz-server-side-encryption-aws-kms-key-id": self.config.kms_key_id},
+            ]
+
+        return self.client.generate_presigned_post(
+            self._get_bucket_name(), key, ExpiresIn=self.config.ttl_signed_urls, Fields=fields, Conditions=conditions
+        )
 
     def _copy(self, key, storage_from: "StorageClient", key_from: str | None = None) -> int:
         key_from = key_from or key
@@ -211,23 +225,21 @@ async def check(self):
         logger.info(f'Check bucket "{storage_bucket}" availability.')
         key = "mindlogger.txt"
 
-        presigned_data = self.generate_presigned_post(storage_bucket, key)
+        presigned_data = self.generate_presigned_post(key)
 
         logger.info(f"Presigned POST fields are following: {presigned_data['fields'].keys()}")
-        file = io.BytesIO(b"")
-        async with httpx.AsyncClient() as client:
-            try:
-                response = await client.post(
-                    presigned_data["url"], data=presigned_data["fields"], files={"file": (key, file)}
-                )
-                if response.status_code == http.HTTPStatus.NO_CONTENT:
-                    logger.info(f"Bucket {storage_bucket} is available.")
-                else:
-                    logger.info(response.content)
-                    raise Exception("File upload error")
-            except httpx.HTTPError as e:
-                logger.info("File upload error")
-                raise e
+        files = {"file": ("test.txt", b"content")}
+
+        try:
+            response = requests.post(presigned_data["url"], data=presigned_data["fields"], files=files)
+            if response.status_code == http.HTTPStatus.NO_CONTENT:
+                logger.info(f"Bucket {storage_bucket} is available.")
+            else:
+                logger.info(response.content)
+                response.raise_for_status()
+        except requests.exceptions.RequestException as e:
+            logger.info(f"File upload error: {e}")
+            raise e
 
     def _check_is_bucket_public(self) -> bool:
         # Check the bucket policy

src/infrastructure/storage/storage_config.py

@@ -1,10 +1,16 @@
+from typing import Self
+
+from pydantic import model_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 from config import CDNSettings
 
 
 class StorageConfig(BaseSettings):
-    """Configuration class for storage service.  Convenience layer between application and object storage."""
+    """
+    Configuration class for storage service.  Convenience layer between application and object storage.
+    This class represents that storage settings to connect with a single object store (i.e., S3 bucket)
+    """
 
     # Custom (S3 or other) endpoint URL
     endpoint_url: str | None = None
@@ -13,6 +19,9 @@ class StorageConfig(BaseSettings):
     secret_key: str | None = None
     access_key: str | None = None
 
+    kms_enabled: bool = False
+    kms_key_id: str | None = None
+
     # Public domain to front storage keys without scheme
     # TODO Default to null??
     domain: str = ""
@@ -27,6 +36,12 @@ class StorageConfig(BaseSettings):
 
     model_config = SettingsConfigDict(extra="ignore")
 
+    @model_validator(mode="after")
+    def validate_settings(self) -> Self:
+        if self.kms_enabled and not self.kms_key_id:
+            raise ValueError("kms_key_id must be set if kms_enabled is True")
+        return self
+
     @classmethod
     def generate_media_settings(cls, cdn_settings: CDNSettings) -> "StorageConfig":
         return cls(
@@ -39,6 +54,8 @@ def generate_media_settings(cls, cdn_settings: CDNSettings) -> "StorageConfig":
             secret_key=cdn_settings.secret_key,
             access_key=cdn_settings.access_key,
             ttl_signed_urls=cdn_settings.ttl_signed_urls,
+            kms_enabled=cdn_settings.bucket_kms_enabled,
+            kms_key_id=cdn_settings.bucket_kms_key_id,
         )
 
     @classmethod
@@ -51,6 +68,8 @@ def generate_answer_settings(cls, cdn_settings: CDNSettings) -> "StorageConfig":
             secret_key=cdn_settings.secret_key,
             access_key=cdn_settings.access_key,
             ttl_signed_urls=cdn_settings.ttl_signed_urls,
+            kms_enabled=cdn_settings.bucket_answer_kms_enabled,
+            kms_key_id=cdn_settings.bucket_answer_kms_key_id,
         )
 
     @classmethod
@@ -63,6 +82,8 @@ def generate_operations_settings(cls, cdn_settings: CDNSettings) -> "StorageConf
             secret_key=cdn_settings.secret_key,
             access_key=cdn_settings.access_key,
             ttl_signed_urls=cdn_settings.ttl_signed_urls,
+            kms_enabled=cdn_settings.bucket_operation_kms_enabled,
+            kms_key_id=cdn_settings.bucket_operation_kms_key_id,
         )
 
     @classmethod
@@ -75,4 +96,6 @@ def generate_logs_settings(cls, cdn_settings: CDNSettings) -> "StorageConfig":
             secret_key=cdn_settings.secret_key,
             access_key=cdn_settings.access_key,
             ttl_signed_urls=cdn_settings.ttl_signed_urls,
+            kms_enabled=cdn_settings.bucket_answer_kms_enabled,
+            kms_key_id=cdn_settings.bucket_answer_kms_key_id,
         )

src/infrastructure/storage/tests/__init__.py

@@ -7,3 +7,5 @@
 MEDIA_OVERRIDE = "this-is-the-media-override"
 OPERATIONS_OVERRIDE = "this-is-the-operations-override"
 ANSWER_OVERRIDE = "this-is-the-answer-override"
+
+KMS_KEY_ID = "arn:aws:kms:test-kms-key"

src/infrastructure/storage/tests/conftest.py

@@ -10,6 +10,7 @@
     ANSWER_BUCKET_NAME,
     ANSWER_OVERRIDE,
     DOMAIN,
+    KMS_KEY_ID,
     MEDIA_BUCKET_NAME,
     MEDIA_OVERRIDE,
     OPERATIONS_BUCKET_NAME,
@@ -137,6 +138,22 @@ async def answer_storage_config(normal_storage_settings: Settings) -> StorageCon
     return config
 
 
+@pytest.fixture
+async def answer_storage_kms_config(normal_storage_settings: Settings) -> StorageConfig:
+    """Settings for storage"""
+    config = StorageConfig(
+        endpoint_url=None,
+        region="us-east-1",
+        bucket=normal_storage_settings.cdn.bucket_answer,
+        access_key=normal_storage_settings.cdn.access_key,
+        secret_key=normal_storage_settings.cdn.secret_key,
+        kms_enabled=True,
+        kms_key_id=KMS_KEY_ID,
+    )
+
+    return config
+
+
 @pytest.fixture
 async def operations_storage_config(normal_storage_settings: Settings) -> StorageConfig:
     """Settings for storage"""

src/infrastructure/storage/tests/test_storage_client.py

@@ -27,6 +27,15 @@ async def answer_storage_client(self, s3_client) -> StorageClient:
 
         return client
 
+    @pytest.fixture
+    async def answer_storage_client_kms(self, s3_client, answer_storage_kms_config) -> StorageClient:
+        """Answer storage client with KMS configured"""
+
+        client = StorageClient(answer_storage_kms_config, env="test")
+        client.client = s3_client
+
+        return client
+
     @pytest.fixture
     async def media_storage_client_other(self, s3_client) -> StorageClient:
         """Regular storage client with local type settings"""
@@ -66,6 +75,23 @@ async def test_generate_presigned_post(self, answer_storage_client):
         assert ANSWER_BUCKET_NAME in data["url"]
         assert ANSWER_OVERRIDE not in data["url"]
 
+        assert "x-amz-server-side-encryption" not in data["fields"]
+        assert "x-amz-server-side-encryption-aws-kms-key-id" not in data["fields"]
+
+    async def test_generate_presigned_post_kms(self, answer_storage_client_kms):
+        data = answer_storage_client_kms.generate_presigned_post(FILE_KEY)
+        assert data is not None
+        assert ANSWER_BUCKET_NAME in data["url"]
+        assert ANSWER_OVERRIDE not in data["url"]
+
+        assert "x-amz-server-side-encryption" in data["fields"]
+        assert "x-amz-server-side-encryption-aws-kms-key-id" in data["fields"]
+
+        assert data["fields"]["x-amz-server-side-encryption"] == "aws:kms"
+        assert (
+            data["fields"]["x-amz-server-side-encryption-aws-kms-key-id"] == answer_storage_client_kms.config.kms_key_id
+        )
+
     async def test_generate_presigned_post_dr(self, answer_storage_client_dr):
         data = answer_storage_client_dr.generate_presigned_post(FILE_KEY)
         assert data is not None
@@ -101,3 +127,8 @@ async def test_get_public_url_other_storage_address(self, media_storage_client_o
         assert DOMAIN not in url
         assert STORAGE_ADDRESS in url
         assert FILE_KEY in url
+
+    @pytest.mark.usefixtures("answer_bucket")
+    async def test_check(self, answer_storage_client, s3_client):
+        answer_storage_client.client = s3_client
+        await answer_storage_client.check()