diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 6fb390bc..64e0684f 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.6.9" + ".": "1.7.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index d93f48a5..d7128ba3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## [1.7.0](https://github.com/equinor/dm-job/compare/v1.6.9...v1.7.0) (2026-08-17) + + +### Features + +* config for selecting resources in azure ci ([2fbf1c7](https://github.com/equinor/dm-job/commit/2fbf1c7b47cd960e741e3f4b8e4c63adeb7ea659)) +* config for selecting resources in azure ci ([695dcd6](https://github.com/equinor/dm-job/commit/695dcd658e6bd2f6abc51e6427cf25551e981c69)) +* config for selecting resources in azure ci ([ee9c5ee](https://github.com/equinor/dm-job/commit/ee9c5ee36b0414d216ee67f3416835b242ed97d2)) + ## [1.6.9](https://github.com/equinor/dm-job/compare/v1.6.8...v1.6.9) (2026-08-04) diff --git a/app/data/WorkflowDS/Blueprints/ComputeResource.json b/app/data/WorkflowDS/Blueprints/ComputeResource.json index 344a6ac2..baa42b57 100644 --- a/app/data/WorkflowDS/Blueprints/ComputeResource.json +++ b/app/data/WorkflowDS/Blueprints/ComputeResource.json @@ -9,8 +9,8 @@ { "name": "memory", "type": "CORE:BlueprintAttribute", - "attributeType": "number", "description": "memory in GB", + "attributeType": "number", "optional": true }, { diff --git a/pyproject.toml b/pyproject.toml index 3cbbff6a..80ec0fb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "dm-job" -version = "1.6.9" # x-release-please-version +version = "1.7.0" # x-release-please-version description = "REST API running jobs in development framework" authors = [ {name = "Equinor", email = "fg_team_hermes@equinor.com"}] license = { text = "MIT" } diff --git a/src/app.py b/src/app.py index 21a15407..038b289b 100644 --- a/src/app.py +++ b/src/app.py @@ -10,9 +10,19 @@ from config import config from features.jobs import jobs +from job_handler_plugins.azure_container_instances import ( + AzureHandlerAuthError, + AzureHandlerConfigError, + AzureHandlerProvisionError, +) from middleware.store_headers import StoreHeadersMiddleware from restful.responses import responses -from utils.exception_handlers import validation_exception_handler +from utils.exception_handlers import ( + azure_auth_exception_handler, + azure_config_exception_handler, + azure_provision_exception_handler, + validation_exception_handler, +) from utils.logging import logger oauth2_scheme = OAuth2AuthorizationCodeBearer(authorizationUrl="", tokenUrl="", auto_error=False) @@ -32,9 +42,14 @@ def create_app(): app = FastAPI( title="Data Modelling Job API", responses=responses, - version="1.6.9", # x-release-please-version + version="1.7.0", # x-release-please-version description="REST API used with the Data Modelling framework to schedule jobs", - exception_handlers={RequestValidationError: validation_exception_handler}, + exception_handlers={ + RequestValidationError: validation_exception_handler, + AzureHandlerConfigError: azure_config_exception_handler, + AzureHandlerAuthError: azure_auth_exception_handler, + AzureHandlerProvisionError: azure_provision_exception_handler, + }, middleware=[Middleware(StoreHeadersMiddleware)], swagger_ui_init_oauth={ "clientId": config.OAUTH_CLIENT_ID, diff --git a/src/job_handler_plugins/azure_container_instances/__init__.py b/src/job_handler_plugins/azure_container_instances/__init__.py index 5cdb90d2..0380a620 100644 --- a/src/job_handler_plugins/azure_container_instances/__init__.py +++ b/src/job_handler_plugins/azure_container_instances/__init__.py @@ -1,10 +1,15 @@ import logging import os +import uuid from collections import namedtuple from time import sleep from typing import Tuple -from azure.core.exceptions import HttpResponseError, ResourceNotFoundError +from azure.core.exceptions import ( + ClientAuthenticationError, + HttpResponseError, + ResourceNotFoundError, +) from azure.identity import ClientSecretCredential from azure.mgmt.containerinstance import ContainerInstanceManagementClient from azure.mgmt.containerinstance.models import ( @@ -28,6 +33,90 @@ _SUPPORTED_TYPE = "dmss://WorkflowDS/Blueprints/AzureContainer" +# Settings that must be present on job-api for this handler to run. +# Only enforced when an AzureContainer job is actually submitted, so deployments +# that use other backends (Radix, LocalContainer, ...) do not need Azure secrets. +_REQUIRED_CONFIG = ( + "AZURE_JOB_SP_CLIENT_ID", + "AZURE_JOB_SP_SECRET", + "AZURE_JOB_SP_TENANT_ID", + "AZURE_JOB_SUBSCRIPTION", + "AZURE_JOB_RESOURCE_GROUP", + "IMAGE_REGISTRY_USERNAME", + "IMAGE_REGISTRY_PASSWORD", +) + + +class AzureHandlerConfigError(RuntimeError): + """Missing or invalid Azure configuration. + + Only raised when an AzureContainer job is actually acted on. Other job + handlers are unaffected, so a deployment that never uses this backend can + run without any Azure secrets being set. + """ + + +class AzureHandlerAuthError(RuntimeError): + """Azure credentials are present but rejected by AAD (expired secret, etc.).""" + + +class AzureHandlerProvisionError(RuntimeError): + """ARM rejected the container-group create/update call. + + Covers quota exhaustion, invalid image references, region capacity, + name collisions, and other non-auth ARM failures. Carries the ARM + status code and error code so the FastAPI boundary can render a + meaningful upstream response. + """ + + def __init__(self, message: str, status_code: int | None = None, error_code: str | None = None): + super().__init__(message) + self.status_code = status_code + self.error_code = error_code + + +def _check_required_config() -> None: + missing = [name for name in _REQUIRED_CONFIG if not getattr(config, name, None)] + if missing: + raise AzureHandlerConfigError( + "The Azure Container Instances handler cannot service this job because " + f"job-api is missing required settings: {', '.join(missing)}. " + "Set them as Radix secrets on the job-api component and redeploy. " + "Other job handlers are unaffected." + ) + + # Cheap sanity check on GUID-shaped fields. AAD would reject these anyway, + # but only after a network round-trip; a local check gives a clearer error. + _GUID_FIELDS = ( + "AZURE_JOB_SP_CLIENT_ID", + "AZURE_JOB_SP_TENANT_ID", + "AZURE_JOB_SUBSCRIPTION", + ) + bad_guids: list[str] = [] + for name in _GUID_FIELDS: + value = getattr(config, name) + try: + uuid.UUID(str(value)) + except (ValueError, TypeError, AttributeError): + bad_guids.append(f"{name}={value!r}") + if bad_guids: + raise AzureHandlerConfigError( + "Azure configuration contains values that are not valid GUIDs: " + f"{', '.join(bad_guids)}. Fix the job-api secrets and redeploy." + ) + + +class _JobLoggerAdapter(logging.LoggerAdapter): + """LoggerAdapter that prepends '[job_uid=]' to every message. + + Callers can log without repeating self.job.job_uid in every f-string, + and log aggregators can group by the prefix. + """ + + def process(self, msg, kwargs): + return f"[job_uid={self.extra['job_uid']}] {msg}", kwargs + + # Interface for Azure @@ -39,17 +128,36 @@ class JobHandler(JobHandlerInterface): def __init__(self, job, data_source: str): super().__init__(job, data_source) - logger.setLevel(logging.WARNING) # I could not find the correctly named logger for this... - azure_credentials = ClientSecretCredential( - client_id=config.AZURE_JOB_SP_CLIENT_ID, - client_secret=config.AZURE_JOB_SP_SECRET, - tenant_id=config.AZURE_JOB_SP_TENANT_ID, - ) + # No config access or SDK construction here. This constructor runs + # whenever the dispatcher touches a job whose runner.type happens to be + # 'AzureContainer' - including status polls for completed jobs - and + # would otherwise crash deployments that only use other backends. self.azure_valid_container_name = self.job.runner["name"].lower().replace(".", "-").replace("_", "-") - self.aci_client = ContainerInstanceManagementClient( - azure_credentials, subscription_id=config.AZURE_JOB_SUBSCRIPTION - ) - logger.setLevel(config.LOGGER_LEVEL) + self._aci_client: ContainerInstanceManagementClient | None = None + self._log = _JobLoggerAdapter(logger, {"job_uid": self.job.job_uid}) + + @property + def aci_client(self) -> ContainerInstanceManagementClient: + """ContainerInstanceManagementClient built on first use. + + Config validation and credential construction are deferred until an + Azure operation is actually needed, so a job-api without Azure secrets + can still service Radix/LocalContainer jobs. + """ + if self._aci_client is None: + _check_required_config() + try: + credentials = ClientSecretCredential( + client_id=config.AZURE_JOB_SP_CLIENT_ID, + client_secret=config.AZURE_JOB_SP_SECRET, + tenant_id=config.AZURE_JOB_SP_TENANT_ID, + ) + except ValueError as exc: # e.g. tenant_id not a valid GUID + raise AzureHandlerConfigError(f"Invalid Azure credential configuration: {exc}") from exc + self._aci_client = ContainerInstanceManagementClient( + credentials, subscription_id=config.AZURE_JOB_SUBSCRIPTION + ) + return self._aci_client def teardown_service(self, service_id: str) -> str: raise NotImplementedError @@ -58,7 +166,7 @@ def setup_service(self, service_id: str) -> str: raise NotImplementedError def start(self) -> str: - logger.info(f"JobName: '{self.job.job_uid}'. Starting Azure Container job...") + self._log.info("Starting Azure Container job...") # Add env-vars from deployment first env_vars: list[EnvironmentVariable] = [ @@ -71,7 +179,7 @@ def start(self) -> str: env_vars.append(EnvironmentVariable(name="JOB_DMSS_ID", value=self.job.dmss_id)) # Parse env-vars from job entity - print("***** Injecting env vars from job entity *****") + logger.info("Injecting env vars from job entity") for env_string in self.job.runner.get("environmentVariables", []): if "=" in env_string: key, value = env_string.split("=", 1) @@ -88,7 +196,9 @@ def start(self) -> str: reference_target: str = self.job.referenceTarget runner_entity: dict = self.job.runner if not runner_entity["image"]["registryName"]: - raise ValueError("Container image in job runner") + raise ValueError( + "Runner entity is missing 'image.registryName'. " f"(runner: {runner_entity.get('name', '')})" + ) full_image_name: str = ( f"{runner_entity['image']['registryName']}/{runner_entity['image']['imageName']}" + f":{runner_entity['image']['version']}" @@ -100,26 +210,24 @@ def start(self) -> str: ) memory_in_gb = 2.0 cpu = 2.0 - if 'computeResource' in runner_entity: - compute_resource = runner_entity['computeResource'] - if 'memory' in compute_resource: - memory_in_gb = compute_resource['memory'] - if 'cpu' in compute_resource: - cpu = compute_resource['cpu'] - if memory_in_gb < 0.5 or cpu < 0.5: - logger.warning( - f"Specified compute resources for job '{self.job.job_uid}' are below the minimum of 0.5 CPU and 0.5 GB memory. " - + f"Using default values of 2 CPU and 2 GB memory." + # ACI Norway East limits (per container group, at time of writing): + # CPU: 0.5 .. 4.0 cores + # Memory: 0.5 .. 16.0 GB + # See: https://learn.microsoft.com/en-us/azure/container-instances/container-instances-region-availability + _CPU_MIN, _CPU_MAX = 0.5, 4.0 + _MEM_MIN, _MEM_MAX = 0.5, 16.0 + if "computeResource" in runner_entity: + compute_resource = runner_entity["computeResource"] + requested_memory = compute_resource.get("memory", memory_in_gb) + requested_cpu = compute_resource.get("cpu", cpu) + memory_in_gb = max(_MEM_MIN, min(_MEM_MAX, float(requested_memory))) + cpu = max(_CPU_MIN, min(_CPU_MAX, float(requested_cpu))) + if memory_in_gb != requested_memory or cpu != requested_cpu: + self._log.warning( + f"Requested compute resources (cpu={requested_cpu}, " + f"memory={requested_memory} GB) clamped to ACI limits " + f"(cpu={cpu}, memory={memory_in_gb} GB)." ) - memory_in_gb = 2.0 - cpu = 2.0 - if memory_in_gb > 16.0 or cpu > 4.0: - logger.warning( - f"Specified compute resources for job '{self.job.job_uid}' are above the maximum of 16 CPU and 4 GB memory. " - + f"Using default values of 2 CPU and 2 GB memory." - ) - memory_in_gb = 2.0 - cpu = 2.0 command_list = ["/app/main/start.sh"] if reference_target: @@ -148,20 +256,41 @@ def start(self) -> str: ) # Create the container group - result = self.aci_client.container_groups.begin_create_or_update( - config.AZURE_JOB_RESOURCE_GROUP, self.azure_valid_container_name, group - ) + try: + result = self.aci_client.container_groups.begin_create_or_update( + config.AZURE_JOB_RESOURCE_GROUP, self.azure_valid_container_name, group + ) - # Wait for the container group to be created and running - # The begin_create_or_update() returns an LROPoller, we need to wait for it to complete - logger.info("Waiting for Azure container group to be provisioned...") - print("Waiting for Azure container group to be provisioned...") - result.result() # This blocks until the operation completes + # Wait for the container group to be created and running + # The begin_create_or_update() returns an LROPoller, we need to wait for it to complete + logger.info("Waiting for Azure container group to be provisioned...") + result.result() # This blocks until the operation completes + except ClientAuthenticationError as exc: + # AADSTS7000215 (invalid secret), 7000222 (expired secret), + # 700016 (unknown app), etc. Surface as a distinct exception so the + # FastAPI boundary can return 502 instead of a bare 500. + raise AzureHandlerAuthError( + "Azure rejected the service principal credentials while starting " + "the container group. The secret is most likely invalid or expired " + f"(check the Radix job-api secrets). AAD detail: {exc.message}" + ) from exc + except HttpResponseError as exc: + # Non-auth ARM failures: quota, invalid image, region capacity, + # name collisions, etc. Carry status/error codes for the API layer. + error_code = getattr(getattr(exc, "error", None), "code", None) + raise AzureHandlerProvisionError( + f"Azure rejected the container-group provisioning request " + f"(container '{self.azure_valid_container_name}'). " + f"ARM status={exc.status_code}, code={error_code}: {exc.message}", + status_code=exc.status_code, + error_code=error_code, + ) from exc # Poll until the container is actually running or has terminated max_wait_seconds = 120 * 5 poll_interval = 5 waited = 0 + container_state: str | None = None while waited < max_wait_seconds: try: container_group = self.aci_client.container_groups.get( @@ -172,87 +301,130 @@ def start(self) -> str: logger.info(f"Container is now in state: {container_state}") break logger.info(f"Container state: {container_state}, waiting...") - print(f"Container state: {container_state}, waiting...") except (AttributeError, TypeError): # instance_view may not be available yet logger.info("Container instance view not yet available, waiting...") - print("Container instance view not yet available, waiting...") except HttpResponseError as e: # Handle ContainerGroupDeploymentNotReady and similar errors if "ContainerGroupDeploymentNotReady" in str(e) or "not ready" in str(e).lower(): logger.info(f"Container group not ready yet: {e.message}") - print(f"Container group not ready yet, waiting. : {e.message}") else: raise # Re-raise if it's a different error sleep(poll_interval) waited += poll_interval + else: + # Loop exited via the while-condition, not via break: we timed out. + raise TimeoutError( + f"Azure container '{self.azure_valid_container_name}' did not " + f"reach Running/Terminated within {max_wait_seconds}s " + f"(last observed state: {container_state!r}). The container " + "group has been created but is stuck - inspect ACI events " + "(image pull, quota, networking) and remove() when done." + ) logger.info("*** Azure container job started successfully ***") - print("*** Azure container job started successfully ***") return "Azure container started" def remove(self) -> Tuple[JobStatus, str]: - logger.setLevel(logging.WARNING) - operation = self.aci_client.container_groups.begin_delete( - config.AZURE_JOB_RESOURCE_GROUP, self.azure_valid_container_name - ) - logger.setLevel(config.LOGGER_LEVEL) + try: + operation = self.aci_client.container_groups.begin_delete( + config.AZURE_JOB_RESOURCE_GROUP, self.azure_valid_container_name + ) + except ResourceNotFoundError: + # Idempotent: already gone counts as successfully removed. + logger.info( + f"Container group '{self.azure_valid_container_name}' already absent; " + "treating remove() as completed." + ) + return JobStatus.COMPLETED, "already removed" + except ClientAuthenticationError as exc: + raise AzureHandlerAuthError( + "Azure rejected the service principal credentials during remove(). " f"AAD detail: {exc.message}" + ) from exc + except HttpResponseError as exc: + error_code = getattr(getattr(exc, "error", None), "code", None) + raise AzureHandlerProvisionError( + f"Azure rejected the container-group delete request " + f"(container '{self.azure_valid_container_name}'). " + f"ARM status={exc.status_code}, code={error_code}: {exc.message}", + status_code=exc.status_code, + error_code=error_code, + ) from exc + + # Poll deletion status status = operation.status() - for i in range(4): + for _ in range(4): status = operation.status() - if status == "Succeeded": + if status in ("Succeeded", "Failed", "Canceled"): break sleep(2) - job_status = JobStatus.UNKNOWN + if status == "Succeeded": - job_status = JobStatus.COMPLETED - return job_status, status + return JobStatus.COMPLETED, status + if status in ("Failed", "Canceled"): + logger.warning(f"Delete of container '{self.azure_valid_container_name}' ended with status={status}") + return JobStatus.FAILED, status + # Still InProgress after the polling budget - not an error, just not done. + return JobStatus.UNKNOWN, status def progress(self) -> Tuple[JobStatus, None | list[str] | str, None | float]: """Poll progress from the job instance""" if self.job.status == JobStatus.FAILED: # If setup fails, the container is not started return self.job.status, self.job.log, self.job.percentage + + # Fetch container group first (cheap, single ARM round-trip). Only pull + # logs if the container has actually reached a state that produces them. try: - logger.setLevel(logging.WARNING) - logs = self.aci_client.containers.list_logs( - config.AZURE_JOB_RESOURCE_GROUP, self.azure_valid_container_name, self.azure_valid_container_name - ).content - logger.setLevel(config.LOGGER_LEVEL) + container_group = self.aci_client.container_groups.get( + config.AZURE_JOB_RESOURCE_GROUP, self.azure_valid_container_name + ) except ResourceNotFoundError: raise NotFoundException( f"The container '{self.azure_valid_container_name}' does not exist. " - + "Either it has not been created, or it's not ready to accept requests." + "Either it has not been created, or it's not ready to accept requests." ) + except ClientAuthenticationError as exc: + raise AzureHandlerAuthError( + "Azure rejected the service principal credentials during progress(). " f"AAD detail: {exc.message}" + ) from exc except HttpResponseError as e: - # Handle ContainerGroupDeploymentNotReady - container is still initializing if "ContainerGroupDeploymentNotReady" in str(e) or "not ready" in str(e).lower(): - logger.info(f"Container group not ready yet for log retrieval: {e}") + logger.info(f"Container group not ready yet: {e}") return JobStatus.STARTING, "Container is still initializing...", self.job.percentage raise try: - container_group = self.aci_client.container_groups.get( - config.AZURE_JOB_RESOURCE_GROUP, self.azure_valid_container_name - ) - status = container_group.containers[0].instance_view.current_state.state - exit_code = container_group.containers[0].instance_view.current_state.exit_code - except HttpResponseError as e: - # Handle ContainerGroupDeploymentNotReady when getting container group status - if "ContainerGroupDeploymentNotReady" in str(e) or "not ready" in str(e).lower(): - logger.info(f"Container group not ready yet: {e}") - return JobStatus.STARTING, "Container is still initializing...", self.job.percentage - raise + current_state = container_group.containers[0].instance_view.current_state + status = current_state.state + exit_code = current_state.exit_code except (AttributeError, TypeError): # instance_view may not be available yet return JobStatus.STARTING, "Container instance view not yet available", self.job.percentage - if not logs: # If no container logs, get the Container Instance events instead + + # Only request logs once the container has content to produce. + logs: None | list[str] | str = None + if status in ("Running", "Terminated"): + try: + logs = self.aci_client.containers.list_logs( + config.AZURE_JOB_RESOURCE_GROUP, + self.azure_valid_container_name, + self.azure_valid_container_name, + ).content + except ResourceNotFoundError: + logs = None + except HttpResponseError as e: + if "ContainerGroupDeploymentNotReady" in str(e) or "not ready" in str(e).lower(): + logger.info(f"Container group not ready yet for log retrieval: {e}") + return JobStatus.STARTING, "Container is still initializing...", self.job.percentage + raise + + if not logs: # Fall back to Container Instance events try: logs = container_group.containers[0].instance_view.events[-1].message - except TypeError: + except (AttributeError, TypeError, IndexError): logs = self.job.log - pass job_status = self.job.status @@ -262,8 +434,18 @@ def progress(self) -> Tuple[JobStatus, None | list[str] | str, None | float]: job_status = JobStatus.RUNNING case ("Terminated", 0): # noqa job_status = JobStatus.COMPLETED - case ("Terminated", exit_code) if exit_code >= 1: # noqa + case ("Terminated", exit_code) if exit_code is not None and exit_code != 0: # noqa + # Includes negative exit codes (SIGKILL, OOM = -9, SIGSEGV = -11, ...) job_status = JobStatus.FAILED case ("Waiting", None): # noqa job_status = JobStatus.STARTING + case ("Succeeded", _): # noqa - ACI provisioning succeeded, container not yet Running + job_status = JobStatus.STARTING + case ("Pending", _): # noqa + job_status = JobStatus.STARTING + case ("Failed", _) | ("Canceled", _): # noqa - ACI-side failures (image pull, quota, ...) + job_status = JobStatus.FAILED + case _: # noqa - any state we haven't mapped + self._log.warning(f"Unmapped ACI container state: status={status!r}, exit_code={exit_code}") + job_status = JobStatus.UNKNOWN return job_status, logs, self.job.percentage diff --git a/src/utils/exception_handlers.py b/src/utils/exception_handlers.py index 7c699938..ab0f815e 100644 --- a/src/utils/exception_handlers.py +++ b/src/utils/exception_handlers.py @@ -4,6 +4,11 @@ from starlette.requests import Request from starlette.responses import JSONResponse +from job_handler_plugins.azure_container_instances import ( + AzureHandlerAuthError, + AzureHandlerConfigError, + AzureHandlerProvisionError, +) from restful.responses import ErrorResponse @@ -18,3 +23,48 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE ).model_dump(), status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, ) + + +async def azure_config_exception_handler(request: Request, exc: AzureHandlerConfigError): + """The AzureContainer handler cannot service this job because job-api is + missing (or has malformed) Azure secrets. Report as 503 - the service + itself is running, but this backend is not configured.""" + return JSONResponse( + ErrorResponse( + status=status.HTTP_503_SERVICE_UNAVAILABLE, + type="AzureHandlerConfigError", + message="Azure Container Instances backend is not configured on job-api.", + debug=str(exc), + ).model_dump(), + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + headers={"Retry-After": "300"}, + ) + + +async def azure_auth_exception_handler(request: Request, exc: AzureHandlerAuthError): + """AAD rejected the service-principal credentials. 502 - upstream refused + authentication; job-api is healthy but Azure said no.""" + return JSONResponse( + ErrorResponse( + status=status.HTTP_502_BAD_GATEWAY, + type="AzureHandlerAuthError", + message="Azure AD rejected the job-api service-principal credentials.", + debug=str(exc), + ).model_dump(), + status_code=status.HTTP_502_BAD_GATEWAY, + ) + + +async def azure_provision_exception_handler(request: Request, exc: AzureHandlerProvisionError): + """ARM rejected the container-group create/delete request for a non-auth + reason (quota, invalid image, region capacity, name collision, ...).""" + return JSONResponse( + ErrorResponse( + status=status.HTTP_502_BAD_GATEWAY, + type="AzureHandlerProvisionError", + message="Azure Resource Manager rejected the container-group operation.", + debug=str(exc), + extra={"arm_status_code": exc.status_code, "arm_error_code": exc.error_code}, + ).model_dump(), + status_code=status.HTTP_502_BAD_GATEWAY, + )