Skip to content

Commit 8c73722

Browse files
gibsondanDagster Devtools
authored andcommitted
Ensure only one thread is ever building the full asset graph at once (#21570)
Summary: A better version of https://github.com/dagster-io/internal/pull/21249 - instead of doing it in one specific context, do it anytime any context is constructing the full asset graph (since it is expensive and we would prefer a bit of contention to having multiple thread doing a heavy asset graph simultaneously) Test Plan: BK ## Summary & Motivation ## Test Plan ## Changelog > The changelog is generated by an agent that examines merged PRs and > summarizes/categorizes user-facing changes. You can optionally replace > this text with a terse description of any user-facing changes in your PR, > which the agent will prioritize. Otherwise, delete this section. Internal-RevId: 19ac17aaed30d702e4c104dc55882ea7dc45b3f5
1 parent b383a06 commit 8c73722

1 file changed

Lines changed: 16 additions & 5 deletions

File tree

python_modules/dagster/dagster/_core/workspace/workspace.py

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1+
import threading
12
from collections.abc import Mapping
23
from enum import Enum
3-
from functools import cached_property
44
from typing import TYPE_CHECKING, Annotated
55

66
from dagster._record import ImportFrom, record
@@ -54,17 +54,28 @@ class CodeLocationStatusEntry:
5454
has_load_error: bool
5555

5656

57-
@record
5857
class CurrentWorkspace:
59-
code_location_entries: Mapping[str, CodeLocationEntry]
58+
def __init__(self, code_location_entries: Mapping[str, CodeLocationEntry]):
59+
self.code_location_entries = code_location_entries
60+
self._asset_graph_lock = threading.Lock()
61+
self._asset_graph: RemoteWorkspaceAssetGraph | None = None
6062

61-
@cached_property
63+
@property
6264
def asset_graph(self) -> "RemoteWorkspaceAssetGraph":
65+
if self._asset_graph is not None:
66+
return self._asset_graph
67+
6368
from dagster._core.definitions.assets.graph.remote_asset_graph import (
6469
RemoteWorkspaceAssetGraph,
6570
)
6671

67-
return RemoteWorkspaceAssetGraph.build(self)
72+
# building the full asset graph is expensive - so ensure at most one
73+
# thread is doing that work at once, and subsequent threads reuse the result
74+
with self._asset_graph_lock:
75+
if self._asset_graph is not None:
76+
return self._asset_graph
77+
self._asset_graph = RemoteWorkspaceAssetGraph.build(self)
78+
return self._asset_graph
6879

6980
def with_code_location(self, name: str, entry: CodeLocationEntry) -> "CurrentWorkspace":
7081
return CurrentWorkspace(code_location_entries={**self.code_location_entries, name: entry})

0 commit comments

Comments
 (0)