22
33from __future__ import annotations
44
5+ import asyncio
56from typing import Any
67
78from orb .application .base .handlers import BaseCommandHandler
1112 CreateReturnRequestCommand ,
1213)
1314from orb .application .ports .query_bus_port import QueryBusPort
15+ from orb .application .services .provider_validation_service import ProviderValidationService
1416from orb .application .services .provisioning_orchestration_service import (
1517 ProvisioningOrchestrationService ,
1618)
2527 LoggingPort ,
2628 ProviderSelectionPort ,
2729)
30+ from orb .domain .request .aggregate import Request
2831from orb .domain .request .request_identifiers import RequestId
32+ from orb .domain .request .request_types import RequestStatus
2933from orb .domain .request .value_objects import RequestType
3034from orb .domain .template .factory import TemplateFactoryPort
3135from orb .domain .template .template_aggregate import Template
@@ -50,6 +54,7 @@ def __init__(
5054 query_bus : QueryBusPort , # QueryBus is required for template lookup
5155 provider_selection_port : ProviderSelectionPort ,
5256 provisioning_service : ProvisioningOrchestrationService ,
57+ provider_validation_service : ProviderValidationService ,
5358 ) -> None :
5459 """Initialize the instance."""
5560 super ().__init__ (logger , event_publisher , error_handler )
@@ -59,7 +64,6 @@ def __init__(
5964 self ._provider_selection_port = provider_selection_port
6065
6166 # Initialize services
62- from orb .application .services .provider_validation_service import ProviderValidationService
6367 from orb .application .services .request_creation_service import RequestCreationService
6468 from orb .application .services .request_status_management_service import (
6569 RequestStatusManagementService ,
@@ -68,9 +72,7 @@ def __init__(
6872 self ._request_creation_service = RequestCreationService (logger )
6973 self ._provisioning_service = provisioning_service
7074 self ._status_service = RequestStatusManagementService (uow_factory , logger )
71- self ._provider_validation_service = ProviderValidationService (
72- container , logger , provider_selection_port
73- )
75+ self ._provider_validation_service = provider_validation_service
7476
7577 async def validate_command (self , command : CreateRequestCommand ) -> None :
7678 """Validate create request command."""
@@ -148,8 +150,6 @@ async def _load_template(self, template_id: str) -> Template:
148150
149151 def _handle_dry_run (self , request : Any ) -> Any :
150152 """Handle dry-run request."""
151- from orb .domain .request .value_objects import RequestStatus
152-
153153 self .logger .info (
154154 "Skipping actual provisioning for request %s (dry-run mode)" ,
155155 request .request_id ,
@@ -167,8 +167,6 @@ async def _persist_and_publish(self, request: Any) -> None:
167167 error is logged at ERROR level but NOT re-raised because the request was
168168 already successfully persisted.
169169 """
170- import asyncio
171-
172170 with self .uow_factory .create_unit_of_work () as uow :
173171 events = uow .requests .save (request )
174172
@@ -268,8 +266,6 @@ async def execute_command(self, command: CreateReturnRequestCommand) -> None:
268266 self .logger .info ("Creating return request for machines: %s" , command .machine_ids )
269267
270268 try :
271- from orb .domain .request .aggregate import Request
272-
273269 domain_config = self ._container .get (DomainConfigurationService )
274270 prefix = domain_config .get_return_request_prefix ()
275271 force_return = command .force_return or False
@@ -297,12 +293,17 @@ async def execute_command(self, command: CreateReturnRequestCommand) -> None:
297293 created_requests : list [str ] = []
298294 pending_deprovision : list [tuple [list [str ], Any , str ]] = []
299295
300- for (provider_type , provider_name ), machine_ids in provider_groups .items ():
296+ for (
297+ provider_type ,
298+ provider_name ,
299+ provider_api ,
300+ ), machine_ids in provider_groups .items ():
301301 return_request_id = str (RequestId .generate (RequestType .RETURN , prefix = prefix ))
302302 request = Request .create_return_request (
303303 machine_ids = machine_ids ,
304304 provider_type = provider_type ,
305305 provider_name = provider_name ,
306+ provider_api = provider_api ,
306307 metadata = command .metadata or {},
307308 request_id = return_request_id ,
308309 )
@@ -330,9 +331,24 @@ async def execute_command(self, command: CreateReturnRequestCommand) -> None:
330331 command .processed_machines = valid_machines
331332 command .skipped_machines = skipped_machines
332333
333- # Await deprovisioning sequentially — one per provider group.
334+ # Synchronous deprovisioning: await provider termination before
335+ # returning. Integration tests + the CLI rely on the handler
336+ # reaching a terminal status by the time it returns; UI-side
337+ # responsiveness is addressed at the API layer (the HTTP
338+ # caller can return early via a 202 + background task at the
339+ # router level, not here in the command handler).
334340 for machine_ids , request , provider_name in pending_deprovision :
335- await self ._execute_deprovisioning_for_request (machine_ids , request , provider_name )
341+ try :
342+ await self ._execute_deprovisioning_for_request (
343+ machine_ids , request , provider_name
344+ )
345+ except Exception as deprov_err :
346+ self .logger .error (
347+ "Deprovisioning failed for request %s: %s" ,
348+ request .request_id ,
349+ deprov_err ,
350+ exc_info = True ,
351+ )
336352
337353 except Exception as e :
338354 self .logger .error (
@@ -436,7 +452,6 @@ async def _update_request_to_in_progress(self, request: Any) -> None:
436452 try :
437453 from orb .application .dto .commands import UpdateRequestStatusCommand
438454 from orb .application .ports .command_bus_port import CommandBusPort
439- from orb .domain .request .request_types import RequestStatus
440455
441456 update_command = UpdateRequestStatusCommand (
442457 request_id = str (request .request_id ),
@@ -481,7 +496,6 @@ async def _execute_deprovisioning_for_request(
481496 if skipped_ids :
482497 from orb .application .dto .commands import UpdateRequestStatusCommand
483498 from orb .application .ports .command_bus_port import CommandBusPort
484- from orb .domain .request .request_types import RequestStatus
485499
486500 skipped_str = ", " .join (skipped_ids )
487501 update_command = UpdateRequestStatusCommand (
@@ -502,11 +516,8 @@ async def _execute_deprovisioning_for_request(
502516 skipped_str ,
503517 )
504518 else :
505- await self ._update_request_to_in_progress_with_message (
506- request ,
507- "Termination initiated, waiting for provider confirmation" ,
508- )
509- self .logger .info ("Termination initiated for request %s" , request .request_id )
519+ await self ._update_request_to_terminating (request )
520+ self .logger .info ("Termination accepted for request %s" , request .request_id )
510521 else :
511522 await self ._update_request_to_failed (request , provisioning_result .get ("errors" , []))
512523
@@ -533,7 +544,9 @@ def _update_machines_to_pending(self, machine_ids: list[str]) -> None:
533544 )
534545 uow .machines .save (updated_machine )
535546
536- def _persist_return_follow_up_context (self , request : Any , provisioning_result : dict [str , Any ]) -> None :
547+ def _persist_return_follow_up_context (
548+ self , request : Any , provisioning_result : dict [str , Any ]
549+ ) -> None :
537550 """Persist durable provider follow-up metadata needed after this process exits."""
538551 provider_data = provisioning_result .get ("provider_data" )
539552 if not isinstance (provider_data , dict ) or not provider_data :
@@ -546,69 +559,87 @@ def _persist_return_follow_up_context(self, request: Any, provisioning_result: d
546559 for event in events or []:
547560 self .event_publisher .publish (event ) # type: ignore[union-attr]
548561
549- async def _update_request_to_failed (self , request : Any , errors : list [str ]) -> None :
550- """Update request status to failed."""
551- try :
552- from orb .application .dto .commands import UpdateRequestStatusCommand
553- from orb .application .ports .command_bus_port import CommandBusPort
554- from orb .domain .request .request_types import RequestStatus
555-
556- error_message = "; " .join (errors ) if errors else "Deprovisioning failed"
562+ async def _update_request_status (
563+ self , request : Request , status : RequestStatus , message : str
564+ ) -> None :
565+ """Update request status via the command bus with a UoW fallback for FAILED.
557566
558- update_command = UpdateRequestStatusCommand (
559- request_id = str (request .request_id ),
560- status = RequestStatus .FAILED ,
561- message = f"Return request failed: { error_message } " ,
562- )
567+ On command-bus failure the error is logged. For FAILED status a direct
568+ UoW write is attempted so the request always reaches a terminal state and
569+ callers do not poll forever. For all other statuses the caller decides
570+ how to handle the failure (e.g. _update_request_to_terminating re-raises
571+ so the deprovisioning loop can apply its own fallback).
572+ """
573+ from orb .application .dto .commands import UpdateRequestStatusCommand
574+ from orb .application .ports .command_bus_port import CommandBusPort
563575
576+ update_command = UpdateRequestStatusCommand (
577+ request_id = str (request .request_id ),
578+ status = status ,
579+ message = message ,
580+ )
581+ try :
564582 command_bus = self ._container .get (CommandBusPort )
565583 await command_bus .execute (update_command )
566-
567- self .logger .info ("Updated request %s status to failed" , request .request_id )
568-
584+ self .logger .info ("Updated request %s status to %s" , request .request_id , status )
569585 except Exception as update_error :
570586 self .logger .error (
571- "Failed to update request status: %s" ,
587+ "Failed to update request %s status to %s: %s" ,
588+ request .request_id ,
589+ status ,
572590 update_error ,
573591 exc_info = True ,
574592 )
575- # Force-write terminal status directly to prevent permanent stuck state
576- try :
577- from orb .domain .request .request_types import RequestStatus
593+ if status == RequestStatus .FAILED :
594+ # Force-write terminal status directly to prevent permanent stuck state
595+ try :
596+ with self .uow_factory .create_unit_of_work () as uow :
597+ stuck_request = uow .requests .get_by_id (request .request_id )
598+ if stuck_request :
599+ stuck_request = stuck_request .update_status (
600+ RequestStatus .FAILED ,
601+ "System error: failed to update status after double failure" ,
602+ force = True ,
603+ )
604+ uow .requests .save (stuck_request )
605+ except Exception as final_error :
606+ self .logger .critical (
607+ "CRITICAL: Failed to mark request %s as failed after double failure. "
608+ "Request is stuck in IN_PROGRESS. Manual intervention required. Error: %s" ,
609+ request .request_id ,
610+ final_error ,
611+ )
612+ else :
613+ raise
578614
579- with self .uow_factory .create_unit_of_work () as uow :
580- stuck_request = uow .requests .get_by_id (request .request_id )
581- if stuck_request :
582- stuck_request = stuck_request .update_status (
583- RequestStatus .FAILED ,
584- "System error: failed to update status after double failure" ,
585- force = True ,
586- )
587- uow .requests .save (stuck_request )
588- except Exception as final_error :
589- self .logger .critical (
590- "CRITICAL: Failed to mark request %s as failed after double failure. "
591- "Request is stuck in IN_PROGRESS. Manual intervention required. Error: %s" ,
592- request .request_id ,
593- final_error ,
594- )
595- # Nothing more we can do
615+ async def _update_request_to_failed (self , request : Any , errors : list [str ]) -> None :
616+ """Update request status to failed."""
617+ error_message = "; " .join (errors ) if errors else "Deprovisioning failed"
618+ await self ._update_request_status (
619+ request , RequestStatus .FAILED , f"Return request failed: { error_message } "
620+ )
621+
622+ async def _update_request_to_terminating (self , request : Any ) -> None :
623+ """Set return request to IN_PROGRESS after termination is accepted by provider.
624+
625+ Termination is asynchronous — the provider accepted the request but instances
626+ are still ``shutting-down``. Using IN_PROGRESS here lets the background sync
627+ poll AWS and transition to COMPLETED only when all instances reach ``terminated``.
628+ """
629+ await self ._update_request_status (
630+ request ,
631+ RequestStatus .IN_PROGRESS ,
632+ "Termination accepted: waiting for instances to reach terminated state" ,
633+ )
596634
597635 async def _update_request_to_in_progress_with_message (self , request : Any , message : str ) -> None :
598636 """Persist an in-progress return message after submit without claiming completion."""
599637 try :
600- from orb .application .dto .commands import UpdateRequestStatusCommand
601- from orb .application .ports .command_bus_port import CommandBusPort
602- from orb .domain .request .request_types import RequestStatus
603-
604- update_command = UpdateRequestStatusCommand (
605- request_id = str (request .request_id ),
606- status = RequestStatus .IN_PROGRESS ,
607- message = message ,
638+ await self ._update_request_status (
639+ request ,
640+ RequestStatus .IN_PROGRESS ,
641+ message ,
608642 )
609- command_bus = self ._container .get (CommandBusPort )
610- await command_bus .execute (update_command )
611- self .logger .info ("Updated request %s status to in_progress" , request .request_id )
612643 except Exception as update_error :
613644 self .logger .error (
614645 "Failed to update request status to in_progress: %s" ,
0 commit comments