fix(provisioning,api): surface handler failures + always-on loopback-admin middleware

aws_provisioning_adapter._provision_via_handlers now checks
result["success"] before returning.  Provider handlers that swallow
exceptions and return {"success": False, "error_message": ...} were
flowing through unchecked: instance_operation_service wrapped the
failure in ProviderResult.success_result, the orchestrator saw
success=True with resource_ids=[], and the request landed on FAILED
with the misleading "No instances provisioned and no cloud resources
created" message instead of the real error.  Re-raising as
InfrastructureError lets _handle_provisioning_failure stamp the
actual cause.

api.server now installs _LoopbackAdminTokenMiddleware unconditionally
and calls _load_loopback_token outside the auth-enabled branch.  The
loopback-admin token written by the daemon at start
(<work_dir>/server/orb-server.token) is the security model for
intra-host admin operations; it must work even when the primary auth
middleware is disabled, so the CLI reload command and the live REST
tests can authenticate as admin via Authorization: Bearer <token>.

tests/providers/aws/live/test_rest_api_onaws ORBServerManager reads
the same token after the /health probe succeeds and threads it
through to RestApiClient.session.headers as Authorization: Bearer
<token>.  Role-guarded routes (POST /machines/request, /machines/return,
admin endpoints) now accept the live REST tests.

Author: fgogolli

src/orb/api/server.py

@@ -117,6 +117,41 @@ def _load_loopback_token(server_config: Any) -> None:
         _server_logger.debug("loopback-admin token load skipped: %s", exc)
 
 
+class _LoopbackAdminTokenMiddleware:
+    """Always-on middleware that stamps admin identity for valid loopback tokens.
+
+    Runs regardless of whether the primary auth middleware is enabled.  When
+    the request carries ``Authorization: Bearer <token>`` and that token
+    matches the value the daemon wrote to ``<work_dir>/server/orb-server.token``
+    at startup, the middleware stamps ``request.state`` with the admin role so
+    role-guarded routes (POST /machines/request, /admin/*, etc.) accept the
+    call.
+
+    Fall-through (no header, or non-matching token) leaves request state
+    untouched so the ``AuthMiddleware`` (when present) and ``get_current_user``
+    dependency continue to behave as before.
+    """
+
+    def __init__(self, app: Any) -> None:
+        from starlette.middleware.base import BaseHTTPMiddleware
+
+        self._impl = BaseHTTPMiddleware(app, dispatch=self._dispatch)
+
+    async def __call__(self, scope: Any, receive: Any, send: Any) -> None:
+        await self._impl(scope, receive, send)
+
+    @staticmethod
+    async def _dispatch(request: Any, call_next: Any) -> Any:
+        auth = request.headers.get("authorization", "")
+        if auth.startswith("Bearer "):
+            candidate = auth[7:].strip()
+            if candidate and candidate in _LOOPBACK_ADMIN_TOKENS:
+                request.state.user_id = "loopback-admin"
+                request.state.user_roles = ["admin"]
+                request.state.permissions = ["*"]
+        return await call_next(request)
+
+
 def create_fastapi_app(server_config: Any) -> Any:
     """
     Create and configure FastAPI application.
@@ -225,13 +260,21 @@ def create_fastapi_app(server_config: Any) -> Any:
     app.add_middleware(LoggingMiddleware)
     logger.info("Logging middleware enabled")
 
+    # Load the daemon-issued loopback-admin token unconditionally so the CLI's
+    # reload command and the live REST tests can authenticate as admin
+    # regardless of whether the primary auth middleware is enabled.  The
+    # token-only middleware below validates Authorization headers and stamps
+    # request.state with the admin role; if no token (or a non-matching token)
+    # is present, request.state is left untouched and the rest of the pipeline
+    # behaves as before.
+    _load_loopback_token(server_config)
+    app.add_middleware(_LoopbackAdminTokenMiddleware)
+    logger.info("Loopback-admin token middleware enabled")
+
     # Add authentication middleware if enabled
     if server_config.auth.enabled:
         auth_strategy = _create_auth_strategy(server_config.auth)
         if auth_strategy:
-            # Load the daemon-issued loopback-admin token (if present) so the
-            # CLI reload command can authenticate without a user-facing JWT.
-            _load_loopback_token(server_config)
             # Wrap the real strategy so loopback tokens are checked first.
             auth_port: Any = _LoopbackAdminAuthWrapper(auth_strategy)
             app.add_middleware(

src/orb/providers/aws/infrastructure/adapters/aws_provisioning_adapter.py

@@ -178,6 +178,19 @@ def _provision_via_handlers(
             # Acquire hosts using the handler — raises typed exception on failure
             result: dict[str, Any] = handler.acquire_hosts(request, aws_template)  # type: ignore[arg-type]
 
+            # Some handler paths swallow exceptions and return a failure dict
+            # (``{"success": False, "error_message": ...}``) instead of raising.
+            # Without this guard, the failure flows through unchecked and
+            # ``instance_operation_service`` wraps it in ``ProviderResult.success_result``,
+            # so the orchestrator sees ``success=True`` with ``resource_ids=[]`` and the
+            # request lands on ``FAILED`` with the misleading "No instances provisioned
+            # and no cloud resources created" message.  Re-raise here so the real error
+            # makes it to ``_handle_provisioning_failure``.
+            if not result.get("success", True):
+                raise InfrastructureError(
+                    result.get("error_message", "Provider handler returned failure")
+                )
+
             resource_ids = result.get("resource_ids", [])
             self._logger.info("Successfully provisioned resources with IDs %s", resource_ids)
             return result

tests/providers/aws/live/test_rest_api_onaws.py

@@ -183,6 +183,10 @@ def __init__(
         self.log_path = log_path
         self._log_file_handle = None
         self._captured_log_path: Optional[str] = None
+        # Loopback-admin token written by the daemon on start; the REST client
+        # reads it and sends `Authorization: Bearer <token>` so the role-guarded
+        # routes (POST /machines/request etc.) accept the request.
+        self.loopback_admin_token: Optional[str] = None
 
     def start(self, timeout: int | None = None):
         """Start ORB server: orb server start --foreground --api-only --host <h> --port <p>"""
@@ -237,6 +241,7 @@ def start(self, timeout: int | None = None):
                 )
                 if response.status_code == 200:
                     log.info("ORB server started successfully on %s", self.base_url)
+                    self.loopback_admin_token = self._read_loopback_token()
                     return
             except requests.exceptions.RequestException:
                 time.sleep(scenarios_rest_api.REST_API_SERVER["start_probe_interval"])
@@ -253,6 +258,34 @@ def start(self, timeout: int | None = None):
         )
         raise RuntimeError(error_msg)
 
+    def _read_loopback_token(self) -> Optional[str]:
+        """Read the loopback-admin bearer token written by ``orb server start``.
+
+        The daemon writes the token next to the pid file at
+        ``<work_dir>/server/orb-server.token`` (mode 0o600).  The REST client
+        sends it as ``Authorization: Bearer <token>`` so role-guarded routes
+        (POST /machines/request, /machines/return, /requests, ...) authenticate
+        as admin.  Without the token the routes return 403 because the
+        anonymous fallback resolves to the ``viewer`` role.
+
+        Returns ``None`` when the token file is missing — falls through to the
+        unauthenticated client (older daemons or daemons started without
+        ``_write_token_file`` permissions).
+        """
+        try:
+            from orb.config.platform_dirs import get_work_location
+            from orb.interface.server_daemon import _token_path
+
+            pid_path = get_work_location() / "server" / "orb-server.pid"
+            token_file = _token_path(pid_path)
+            if not token_file.exists():
+                log.warning("Loopback admin token file not found at %s", token_file)
+                return None
+            return token_file.read_text(encoding="ascii").strip() or None
+        except Exception as exc:
+            log.warning("Failed to read loopback admin token: %s", exc)
+            return None
+
     def _read_captured_output(self) -> str:
         """Read the last 2KB of the server log for error context."""
         try:
@@ -299,12 +332,18 @@ def __init__(
         timeout: int | None = None,
         api_prefix: str = scenarios_rest_api.REST_API_PREFIX,
         retry_attempts: int | None = None,
+        auth_token: str | None = None,
     ):
         self.base_url = base_url.rstrip("/")
         self.api_prefix = api_prefix
         self.timeout = timeout or REST_TIMEOUTS["rest_api_timeout"]
         self.retry_attempts = retry_attempts or REST_TIMEOUTS["rest_api_retry_attempts"]
         self.session = requests.Session()
+        if auth_token:
+            # The token is the loopback-admin bearer secret written by the
+            # daemon at start; required for role-guarded routes when auth
+            # is enabled OR when auth is disabled (anonymous → viewer).
+            self.session.headers["Authorization"] = f"Bearer {auth_token}"
 
     def _url(self, path: str) -> str:
         """Construct full URL with API prefix."""
@@ -483,6 +522,7 @@ def rest_api_client(orb_server):
         api_prefix="/api/v1",
         timeout=REST_TIMEOUTS["rest_api_timeout"],
         retry_attempts=REST_TIMEOUTS["rest_api_retry_attempts"],
+        auth_token=orb_server.loopback_admin_token,
     )
 
 
@@ -2239,6 +2279,7 @@ def test_rest_api_unknown_template_returns_error(setup_rest_api_environment):
             base_url=server.base_url,
             api_prefix="/api/v1",
             timeout=REST_TIMEOUTS["rest_api_timeout"],
+            auth_token=server.loopback_admin_token,
         )
         try:
             client.request_machines("NonExistent-Template-XYZ", 1)