Encrypt and upload Bigpicture XML files to S3inbox upon publishing (merge commit)

Merge branch 'feature/encrypt-and-send-xmls' into 'main'
* Addressed comments

* Encrypt and send BP XML files to S3inbox

Closes #929
See merge request https://gitlab.ci.csc.fi/sds-dev/sd-submit/metadata-submitter/-/merge_requests/1126

Approved-by: Rasko Leinonen <raskolei@csc.fi>
Co-authored-by: Joonatan Mäkinen <joonatan.makinen@csc.fi>
Merged by Rasko Leinonen <raskolei@csc.fi>

Author: Rasko Leinonen

Makefile

@@ -53,9 +53,6 @@ get_env: ## Get secrets needed for integration tests from vault
 	$(call write_secret,SDS_AAI_CLIENT_ID,sd-submit/secrets,sds_aai_id) \
 	$(call write_secret,SDS_AAI_CLIENT_SECRET,sd-submit/secrets,sds_aai_secret) \
 	$(call write_secret,SDS_AAI_URL,sd-submit/secrets,sds_aai_url) \
-	$(call write_secret,LS_AAI_CLIENT_ID,sd-submit/secrets,ls_aai_id) \
-	$(call write_secret,LS_AAI_CLIENT_SECRET,sd-submit/secrets,ls_aai_secret) \
-	$(call write_secret,LS_AAI_URL,sd-submit/secrets,ls_aai_url) \
 	$(call write_secret,KEYSTONE_ENDPOINT,sd-submit/secrets,pouta_host) \
 	$(call write_secret,NBIS_JWT_PUBLIC_KEY,sd-submit/secrets,nbis_jwt_public_key) \
 	$(call write_integration_test_secret,DATACITE_API,sd-submit/datacite_test,DOI_API) \

docker-compose.yml

@@ -75,9 +75,6 @@ services:
       start_period: 30s
     environment:
       - "DEPLOYMENT=NBIS"
-      - "OIDC_CLIENT_SECRET=${LS_AAI_CLIENT_SECRET:-${OIDC_CLIENT_SECRET:?OIDC_CLIENT_SECRET must be defined}}"
-      - "OIDC_CLIENT_ID=${LS_AAI_CLIENT_ID:-${OIDC_CLIENT_ID:?OIDC_CLIENT_ID must be defined}}"
-      - "OIDC_URL=${LS_AAI_URL:-${OIDC_URL:?OIDC_URL must be defined}}"
       - "BASE_URL=http://sd-submit-api-nbis:5431"
       - "OIDC_REDIRECT_URL=${OIDC_REDIRECT_URL}"
       - "OIDC_SECURE_COOKIE=${OIDC_SECURE_COOKIE}"
@@ -98,9 +95,12 @@ services:
       - "DATABASE_URL=${NBIS_DATABASE_URL}"
       - "BP_CENTER_ID=${BP_CENTER_ID}"
       - "S3_REGION=${S3_REGION}"
-      - "S3_ENDPOINT=${S3_ENDPOINT}"
+      - "S3_ENDPOINT=${S3_INBOX_ENDPOINT:?S3_INBOX_ENDPOINT must be defined}"
       - "ADMIN_URL=${SDA_API_URL}"
       - "ADMIN_TOKEN=${ADMIN_TOKEN}"
+      - "CRYPT4GH_PUBLIC_KEY=${C4GH_RECIPIENT_PUBLIC_KEY:?C4GH_RECIPIENT_PUBLIC_KEY must be defined}"
+      - "CRYPT4GH_PRIVATE_KEY=${C4GH_SENDER_SECRET_KEY:?C4GH_SENDER_SECRET_KEY must be defined}"
+      - "CRYPT4GH_PRIVATE_KEY_PASSPHRASE=${C4GH_SECRET_KEY_PASSPHRASE:?C4GH_SECRET_KEY_PASSPHRASE must be defined}"
       - "ALLOW_UNSAFE=${ALLOW_UNSAFE}"
 
   mock-oauth2:
@@ -205,6 +205,12 @@ services:
       - "8006:8006"
     environment:
       - MOTO_PORT=8006
+    healthcheck:
+      test: curl --fail http://127.0.0.1:8006/ || exit 1
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 10s
 
   postgres-csc:
     image: postgres:18-alpine
@@ -272,3 +278,5 @@ services:
         condition: service_healthy
       mockldap:
         condition: service_started
+      mockinbox:
+        condition: service_healthy

metadata_backend/api/handlers/publish.py

@@ -14,8 +14,12 @@
 from ..models.datacite import DataCiteMetadata
 from ..models.models import File, Registration, SubmissionId
 from ..models.submission import Rems, Submission, SubmissionMetadata, SubmissionWorkflow
-from ..processors.xml.bigpicture import BP_POLICY_OBJECT_TYPE, BP_XML_OBJECT_CONFIG
+from ..processors.xml.bigpicture import (
+    BP_POLICY_OBJECT_TYPE,
+    BP_XML_OBJECT_CONFIG,
+)
 from ..processors.xml.processors import XmlObjectProcessor
+from ..services.bigpicture import upload_bp_metadata_xmls
 from ..services.datacite import DataciteService
 from ..services.submission.bigpicture import is_clinical_policy
 from .restapi import RESTAPIHandler
@@ -139,7 +143,7 @@ async def _publish_rems(self, submission: Submission, rems: Rems, registration:
             # Create REMS resource.
             if not registration.remsResourceId:
                 if submission.workflow == SubmissionWorkflow.BP:
-                    # Use BigPicture dataset id as the REMS resource id.
+                    # Use Bigpicture dataset id as the REMS resource id.
                     resid = submission.submissionId
                 elif registration.doi is not None:
                     # Use DOI as the REMS resource id.
@@ -304,6 +308,13 @@ async def publish_submission(
         if deployment_config().ALLOW_REGISTRATION:
             await self._register_submission(submission, datacite, rems)
 
+        if workflow == SubmissionWorkflow.BP:
+            headers = req.headers
+            jwt = self._services.auth._get_bearer_token(headers)
+            if not jwt:
+                raise UserException("Missing OIDC access token in Authorization bearer header for SDA inbox upload.")
+            await upload_bp_metadata_xmls(self._services, submission_id, user.user_id, jwt)
+
         # Update submission status to published.
         await submission_service.publish(submission_id)
 

metadata_backend/api/handlers/restapi.py

@@ -49,7 +49,7 @@ class RESTAPIServiceHandlers(BaseModel):
     ror: RorServiceHandler | None
     rems: RemsServiceHandler
     keystone: KeystoneServiceHandler | None
-    auth: AuthServiceHandler
+    auth: AuthServiceHandler | None
     admin: AdminServiceHandler | None = None
     database: HealthHandler
 

metadata_backend/api/services/auth.py

@@ -10,6 +10,7 @@
 import jwt
 from fastapi import HTTPException
 from starlette import status
+from starlette.datastructures import Headers
 
 from ...conf.jwt import jwt_config
 from ...database.postgres.models import ApiKeyEntity
@@ -219,3 +220,20 @@ async def list_api_keys(self, user_id: str) -> list[ApiKey]:
         """
 
         return await self.__repository.get_api_keys(user_id)
+
+    @staticmethod
+    def _get_bearer_token(headers: Headers) -> str | None:
+        """Get OIDC access token from Authorization bearer header.
+
+        Args:
+            headers: The HTTP headers containing the Authorization bearer token.
+
+        Returns:
+            The OIDC access token if present, None otherwise.
+        """
+        auth_header = headers.get("Authorization", "")
+        if auth_header.lower().startswith("bearer "):
+            token = auth_header[7:].strip()
+            if token:
+                return token
+        return None

metadata_backend/api/services/bigpicture.py

@@ -0,0 +1,112 @@
+"""Bigpicture API services."""
+
+from ..exceptions import SystemException
+from ..handlers.restapi import RESTAPIServices
+from ..processors.xml.bigpicture import (
+    BP_ANNOTATION_OBJECT_TYPE,
+    BP_DATASET_OBJECT_TYPE,
+    BP_IMAGE_OBJECT_TYPE,
+    BP_LANDING_PAGE_OBJECT_TYPE,
+    BP_OBSERVATION_OBJECT_TYPE,
+    BP_OBSERVER_OBJECT_TYPE,
+    BP_ORGANISATION_OBJECT_TYPE,
+    BP_POLICY_OBJECT_TYPE,
+    BP_REMS_OBJECT_TYPE,
+    BP_SAMPLE_OBJECT_TYPES,
+    BP_STAINING_OBJECT_TYPE,
+)
+from .file import S3InboxSDAService
+
+BP_METADATA_FILES: tuple[tuple[str | tuple[str, ...], str], ...] = (
+    (BP_DATASET_OBJECT_TYPE, "dataset.xml.c4gh"),
+    (BP_POLICY_OBJECT_TYPE, "policy.xml.c4gh"),
+    (BP_IMAGE_OBJECT_TYPE, "image.xml.c4gh"),
+    (BP_ANNOTATION_OBJECT_TYPE, "annotation.xml.c4gh"),
+    (BP_OBSERVATION_OBJECT_TYPE, "observation.xml.c4gh"),
+    (BP_OBSERVER_OBJECT_TYPE, "observer.xml.c4gh"),
+    (tuple(BP_SAMPLE_OBJECT_TYPES), "sample.xml.c4gh"),
+    (BP_STAINING_OBJECT_TYPE, "staining.xml.c4gh"),
+)
+
+BP_LANDING_PAGE: tuple[tuple[str, str], ...] = ((BP_LANDING_PAGE_OBJECT_TYPE, "landing_page.xml.c4gh"),)
+
+BP_PRIVATE_METDATA_FILES: tuple[tuple[str, str], ...] = (
+    (BP_ORGANISATION_OBJECT_TYPE, "organisation.xml.c4gh"),
+    (BP_REMS_OBJECT_TYPE, "rems.xml.c4gh"),
+)
+
+
+async def _upload_xml_documents(
+    services: RESTAPIServices,
+    file_provider: S3InboxSDAService,
+    *,
+    bucket_name: str,
+    prefix: str,
+    object_files: tuple[tuple[str | tuple[str, ...], str], ...],
+    user_id: str,
+    jwt: str,
+    submission_id: str,
+) -> None:
+    for object_type, filename in object_files:
+        xml = None
+        async for xml_doc in services.object.get_xml_documents(submission_id, object_type):
+            xml = xml_doc
+            break
+        if xml is None:
+            continue
+
+        object_key = f"{prefix}/{filename}"
+        await file_provider._add_file_to_bucket(
+            bucket_name=bucket_name,
+            object_key=object_key,
+            access_key=user_id,
+            secret_key=user_id,
+            session_token=jwt,
+            body=xml.encode("utf-8"),
+        )
+
+
+async def upload_bp_metadata_xmls(services: RESTAPIServices, submission_id: str, user_id: str, jwt: str) -> None:
+    """Upload encrypted Bigpicture metadata XML files to SDA inbox."""
+    file_provider = services.file_provider
+    if not isinstance(file_provider, S3InboxSDAService):
+        raise SystemException("Bigpicture metadata upload requires SDA inbox file provider service.")
+
+    bucket = user_id.replace("@", "_")  # SDA inbox bucket name is the user id with @ replaced by underscore
+
+    # Metadata XML files
+    await _upload_xml_documents(
+        services,
+        file_provider,
+        bucket_name=bucket,
+        prefix=f"DATASET_{submission_id}/METADATA",
+        object_files=BP_METADATA_FILES,
+        user_id=user_id,
+        jwt=jwt,
+        submission_id=submission_id,
+    )
+
+    # Landing page XML file
+    await _upload_xml_documents(
+        services,
+        file_provider,
+        bucket_name=bucket,
+        prefix=f"DATASET_{submission_id}/LANDING_PAGE",
+        object_files=BP_LANDING_PAGE,
+        user_id=user_id,
+        jwt=jwt,
+        submission_id=submission_id,
+    )
+
+    # Private metadata XML files
+    # TODO(improve): Add datacite.xml to private metadata files once datacite.xml is available
+    await _upload_xml_documents(
+        services,
+        file_provider,
+        bucket_name=bucket,
+        prefix=f"DATASET_{submission_id}/PRIVATE",
+        object_files=BP_PRIVATE_METDATA_FILES,
+        user_id=user_id,
+        jwt=jwt,
+        submission_id=submission_id,
+    )

metadata_backend/api/services/file.py

@@ -1,12 +1,18 @@
 """Service to retrieve file and bucket information from a file provider."""
 
+import base64
+import binascii
 from abc import ABC, abstractmethod
+from io import BytesIO
 
 import aioboto3
 import botocore.exceptions
 import ujson
+from crypt4gh.keys import c4gh
+from crypt4gh.lib import encrypt
 from pydantic import BaseModel, RootModel
 
+from ...conf.c4gh import c4gh_config
 from ...conf.s3 import s3_config
 from ...helpers.logger import LOG
 from ...services.admin_service import AdminServiceHandler
@@ -389,7 +395,6 @@ async def _verify_bucket_policy(self, bucket: str) -> bool:
         return False
 
 
-# WIP: to be possibly used for writing XML file to SDA S3 Inbox.
 class S3InboxSDAService(FileProviderService):
     """Service to manage S3 buckets in NeIC SDA S3 Inbox."""
 
@@ -433,30 +438,85 @@ async def check_files_exist(self, user_id: str, files: list[SubmissionFile]) ->
         inbox_paths = [inbox_file.get("inboxPath", "") for inbox_file in inbox_files]
         return [file.path for file in files if not any(file.path in inbox_path for inbox_path in inbox_paths)]
 
+    async def _load_crypt4gh_keys(self) -> tuple[object, object]:
+        """Load Crypt4GH sender secret and recipient public keys from env variables."""
+        conf = c4gh_config()
+        try:
+            sender_key_pem = base64.b64decode(conf.CRYPT4GH_PRIVATE_KEY).decode("utf-8")
+            recipient_key_pem = base64.b64decode(conf.CRYPT4GH_PUBLIC_KEY).decode("utf-8")
+        except (binascii.Error, UnicodeDecodeError) as ex:
+            raise SystemException("Invalid base64 in C4GH key environment variables.") from ex
+
+        try:
+            sender_lines = [line.strip().encode("utf-8") for line in sender_key_pem.splitlines() if line.strip()]
+            recipient_lines = [line.strip().encode("utf-8") for line in recipient_key_pem.splitlines() if line.strip()]
+
+            private_data = base64.b64decode(b"".join(sender_lines[1:-1]))
+            public_data = base64.b64decode(b"".join(recipient_lines[1:-1]))
+
+            private_stream = BytesIO(private_data)
+            if private_data.startswith(c4gh.MAGIC_WORD):
+                private_stream.seek(len(c4gh.MAGIC_WORD))
+
+            sender_secret_key = c4gh.parse_private_key(private_stream, lambda: conf.CRYPT4GH_PRIVATE_KEY_PASSPHRASE)
+            recipient_public_key = public_data
+            return sender_secret_key, recipient_public_key
+        except Exception as ex:
+            raise SystemException("Failed to load Crypt4GH keys for Bigpicture metadata encryption.") from ex
+
+    async def _encrypt_file(self, file: bytes, sender_secret_key: object, recipient_public_key: object) -> bytes:
+        """Encrypt file bytes using crypt4gh and return encrypted payload bytes."""
+        infile = BytesIO(file)
+        outfile = BytesIO()
+        encrypt([(0, sender_secret_key, recipient_public_key)], infile, outfile)
+        return outfile.getvalue()
+
     async def _add_file_to_bucket(
-        self, bucket_name: str, object_key: str, access_key: str, secret_key: str, session_token: str
+        self,
+        bucket_name: str,
+        object_key: str,
+        access_key: str,
+        secret_key: str,
+        session_token: str,
+        body: bytes = b"",
     ) -> None:
-        """Add a new object to S3 bucket using provided credentials.
+        """Put a C4GH encrypted object to S3 bucket using provided credentials.
 
         :param bucket_name: name of the bucket
         :param object_key: key for the object to be added
         :param access_key: S3 access key ID
         :param secret_key: S3 secret access key
         :param session_token: S3 session token
+        :param body: unencrypted object bytes
         """
-        # TODO(improve): Add file content as body instead of empty file.
-        session = aioboto3.Session()
-        async with session.client(
-            "s3",
-            endpoint_url=self.endpoint,
-            aws_access_key_id=access_key,
-            aws_secret_access_key=secret_key,
-            aws_session_token=session_token,  # equivalent to s3cmd access_token
-            region_name=self.region,
-        ) as s3:
-            await s3.put_object(
-                Bucket=bucket_name,
-                Key=object_key,
-                Body="",
-                ContentType="application/json",
+        sender_secret_key, recipient_public_key = await self._load_crypt4gh_keys()
+        encrypted_file = await self._encrypt_file(body, sender_secret_key, recipient_public_key)
+
+        try:
+            session = aioboto3.Session()
+            async with session.client(
+                "s3",
+                endpoint_url=self.endpoint,
+                aws_access_key_id=access_key,
+                aws_secret_access_key=secret_key,
+                aws_session_token=session_token,  # equivalent to s3cmd access_token
+                region_name=self.region,
+            ) as s3:
+                await s3.put_object(
+                    Bucket=bucket_name,
+                    Key=object_key,
+                    Body=encrypted_file,
+                    ContentType="application/octet-stream",
+                )
+        except botocore.exceptions.ClientError as ex:
+            err = ex.response.get("Error", {})
+            code = err.get("Code")
+            msg = err.get("Message")
+            LOG.exception(
+                "Failed to upload encrypted file to SDA inbox bucket '%s' key '%s': %s - %s",
+                bucket_name,
+                object_key,
+                code,
+                msg,
             )
+            raise SystemException("Failed to upload encrypted file to SDA inbox.") from ex

metadata_backend/api/services/submission/bigpicture.py

@@ -78,7 +78,7 @@ def is_clinical_policy(policy_processor: XmlObjectProcessor) -> bool:
 
 
 class BigpictureObjectSubmissionService(ObjectSubmissionService):
-    """Service for processing BigPicture submissions."""
+    """Service for processing Bigpicture submissions."""
 
     def __init__(
         self,
@@ -140,7 +140,7 @@ def _create_processor(objects: list[ObjectSubmission]) -> tuple[XmlStringDocumen
         if datacite_object:
             datacite = read_datacite_xml(datacite_object.document)
 
-        # Create processor for BigPicture XMLs.
+        # Create processor for Bigpicture XMLs.
         processor = XmlStringDocumentsProcessor(BP_XML_OBJECT_CONFIG, [o.document for o in bp_objects])
         return processor, datacite