Skip to content

Commit d068854

Browse files
sndagsterDagster Devtools
authored andcommitted
fix: send deployment header from serverless presigned-URL IO manager (#25897)
## Summary Materializing assets in a serverless org with **more than one full deployment** fails with a 401 from the IO manager: ``` requests.exceptions.HTTPError: 401 Client Error: Unauthorized for url: https://<org>.agent.<env>.dagster.cloud/gen_io_storage_url?key=raw_orders&method=PUT File ".../dagster_cloud/serverless/io_manager.py", line 138, in _get_presigned_url resp.raise_for_status() ``` **Root cause.** `ServerlessPresignedURLIOManager._get_presigned_url` builds its headers with the raw `get_dagster_cloud_api_headers(token, DagsterCloudInstanceScope.DEPLOYMENT)` and **never passes a `deployment_name`**, so the `Dagster-Cloud-Deployment` header is omitted. Ursula's auth (`_get_deployment`) then falls back to "if the org has exactly one deployment, use it; otherwise 401": ``` Organization has multiple deployments, but no deployment specified ``` Single-deployment serverless orgs hit the one-deployment branch and worked; with per-deployment tenancy an org can now have `prod` + `stage`, so it 401s. Every other run-pod callback (compute logs, insights, defs-state, GraphQL) goes through `instance.dagster_cloud_api_headers` → `get_agent_headers`, which derives `deployment_name` from the instance config. This IO manager was the only path that bypassed that helper. ## Fix Thread `init_context.instance.deployment_name` into the manager and send it in the header — matching how the non-presigned S3 path already reads `instance.deployment_name`, and how every other caller sets the header. ## Test Plan - `dagster_cloud_tests/serverless_tests/test_io_manager.py` → 13 passed. Added an assertion that `dump_to_path` sends `Dagster-Cloud-Deployment`, and that the factory threads `deployment_name` onto the manager. - `just ty` → 0 errors. ## Changelog Fix a 401 from the serverless IO manager when materializing assets in an org with more than one full deployment. Internal-RevId: 724b9db96b53cbc581e9460aa2038113b8b7176e
1 parent 2f4af03 commit d068854

2 files changed

Lines changed: 12 additions & 3 deletions

File tree

python_modules/dagster-cloud/dagster_cloud/serverless/io_manager.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,10 @@ class ServerlessPresignedURLIOManager(UPathIOManager):
108108
obtained from the Dagster+ API on each operation.
109109
"""
110110

111-
def __init__(self, api_url: str, api_token: str, timeout: int):
111+
def __init__(self, api_url: str, api_token: str, timeout: int, deployment_name: str | None):
112112
self._api_url = api_url
113113
self._api_token = api_token
114+
self._deployment_name = deployment_name
114115
self._timeout = timeout
115116
self._session = requests.Session()
116117
put_retry_adapter = HTTPAdapter(
@@ -131,6 +132,7 @@ def _get_presigned_url(self, key: str, method: str) -> str:
131132
headers=get_dagster_cloud_api_headers(
132133
self._api_token,
133134
DagsterCloudInstanceScope.DEPLOYMENT,
135+
deployment_name=self._deployment_name,
134136
),
135137
params={"key": key, "method": method},
136138
timeout=self._timeout,
@@ -182,6 +184,7 @@ def _build_serverless_io_manager(init_context):
182184
api_url=init_context.instance.dagster_cloud_url,
183185
api_token=init_context.instance.dagster_cloud_agent_token,
184186
timeout=init_context.instance.dagster_cloud_api_timeout,
187+
deployment_name=init_context.instance.deployment_name,
185188
)
186189

187190
bucket = os.getenv("DAGSTER_CLOUD_SERVERLESS_STORAGE_S3_BUCKET")

python_modules/dagster-cloud/dagster_cloud_tests/serverless_tests/test_io_manager.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,13 @@ def test_factory_returns_presigned_when_env_var_set():
5858
init_context.instance.dagster_cloud_url = "https://myorg.agent.dagster.cloud"
5959
init_context.instance.dagster_cloud_agent_token = "test-token"
6060
init_context.instance.dagster_cloud_api_timeout = 30
61+
init_context.instance.deployment_name = "prod"
6162

6263
with environ({"SERVERLESS_IO_MANAGER_USE_PRESIGNED_URL": "True"}):
6364
manager = _build_serverless_io_manager(init_context)
6465

6566
assert isinstance(manager, ServerlessPresignedURLIOManager)
67+
assert manager._deployment_name == "prod"
6668

6769

6870
def test_factory_returns_s3_when_env_var_not_set():
@@ -93,12 +95,15 @@ def test_factory_returns_s3_when_env_var_not_set():
9395

9496
API_URL = "https://myorg.agent.dagster.cloud"
9597
API_TOKEN = "test-token"
98+
DEPLOYMENT_NAME = "prod"
9699
PRESIGNED_PUT_URL = "https://s3.amazonaws.com/bucket/key?X-Amz-Signature=put123"
97100
PRESIGNED_GET_URL = "https://s3.amazonaws.com/bucket/key?X-Amz-Signature=get123"
98101

99102

100103
def _make_manager() -> ServerlessPresignedURLIOManager:
101-
return ServerlessPresignedURLIOManager(api_url=API_URL, api_token=API_TOKEN, timeout=30)
104+
return ServerlessPresignedURLIOManager(
105+
api_url=API_URL, api_token=API_TOKEN, timeout=30, deployment_name=DEPLOYMENT_NAME
106+
)
102107

103108

104109
def _presigned_url_response(url: str):
@@ -145,6 +150,7 @@ def test_dump_to_path_calls_put():
145150
"key": "storage/run-123/my_step/result",
146151
"method": "PUT",
147152
}
153+
assert call_kwargs.kwargs["headers"]["Dagster-Cloud-Deployment"] == DEPLOYMENT_NAME
148154

149155
mock_put.assert_called_once()
150156
(put_url,) = mock_put.call_args.args
@@ -331,7 +337,7 @@ def test_module_imports_without_boto3_installed():
331337
assert reloaded.ServerlessPresignedURLIOManager is not None
332338
# Instantiating the presigned-URL manager must not touch boto3.
333339
manager = reloaded.ServerlessPresignedURLIOManager(
334-
api_url=API_URL, api_token=API_TOKEN, timeout=30
340+
api_url=API_URL, api_token=API_TOKEN, timeout=30, deployment_name=DEPLOYMENT_NAME
335341
)
336342
assert manager is not None
337343
finally:

0 commit comments

Comments
 (0)