Skip to content

Commit 012ba9e

Browse files
prhaclaude
andcommitted
Add asset graph integration for partitioned asset checks
This PR integrates partitioned asset checks with the asset graph system: - Update AssetGraph to handle partitioned check nodes - Add subset computation logic in AssetGraphView for check partitions - Implement compute_subset_with_status for different execution statuses - Update entity_subset to support asset check partition subsets - Tests validate subset computation works correctly for partitioned and non-partitioned checks 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
1 parent a2ea5eb commit 012ba9e

6 files changed

Lines changed: 168 additions & 51 deletions

File tree

python_modules/dagster/dagster/_core/asset_graph_view/asset_graph_view.py

Lines changed: 31 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
from dagster._core.definitions.asset_key import AssetCheckKey, AssetKey, EntityKey, T_EntityKey
1919
from dagster._core.definitions.assets.graph.asset_graph_subset import AssetGraphSubset
2020
from dagster._core.definitions.events import AssetKeyPartitionKey
21-
from dagster._core.definitions.freshness import FreshnessState
2221
from dagster._core.definitions.partitions.context import (
2322
PartitionLoadingContext,
2423
use_partition_loading_context,
@@ -162,10 +161,7 @@ def get_inner_queryer_for_back_compat(self) -> "CachingInstanceQueryer":
162161
return self._queryer
163162

164163
def _get_partitions_def(self, key: T_EntityKey) -> Optional["PartitionsDefinition"]:
165-
if isinstance(key, AssetKey):
166-
return self.asset_graph.get(key).partitions_def
167-
else:
168-
return None
164+
return self.asset_graph.get(key).partitions_def
169165

170166
@cached_method
171167
@use_partition_loading_context
@@ -550,6 +546,11 @@ async def compute_subset_with_status(
550546
from dagster._core.storage.event_log.base import AssetCheckSummaryRecord
551547

552548
"""Returns the subset of an asset check that matches a given status."""
549+
# Handle partitioned asset checks with partition-level granularity
550+
if self._get_partitions_def(key):
551+
return await self._get_partitioned_check_subset_with_status(key, status)
552+
553+
# Handle non-partitioned asset checks with existing logic
553554
summary = await AssetCheckSummaryRecord.gen(self, key)
554555
latest_record = summary.last_check_execution_record if summary else None
555556
resolved_status = (
@@ -562,26 +563,6 @@ async def compute_subset_with_status(
562563
else:
563564
return self.get_empty_subset(key=key)
564565

565-
async def compute_subset_with_freshness_state(
566-
self, key: AssetKey, state: FreshnessState
567-
) -> EntitySubset[AssetKey]:
568-
from dagster._core.definitions.asset_health.asset_freshness_health import (
569-
AssetFreshnessHealthState,
570-
)
571-
572-
if not self.asset_graph.has(key) or self.asset_graph.get(key).freshness_policy is None:
573-
if state == FreshnessState.NOT_APPLICABLE:
574-
return self.get_full_subset(key=key)
575-
else:
576-
return self.get_empty_subset(key=key)
577-
578-
asset_freshness_health_state = await AssetFreshnessHealthState.compute_for_asset(key, self)
579-
580-
if asset_freshness_health_state.freshness_state == state:
581-
return self.get_full_subset(key=key)
582-
else:
583-
return self.get_empty_subset(key=key)
584-
585566
async def _compute_run_in_progress_check_subset(
586567
self, key: AssetCheckKey
587568
) -> EntitySubset[AssetCheckKey]:
@@ -609,6 +590,31 @@ async def _compute_missing_check_subset(
609590
) -> EntitySubset[AssetCheckKey]:
610591
return await self.compute_subset_with_status(key, None)
611592

593+
async def _get_partitioned_check_subset_with_status(
594+
self, key: AssetCheckKey, status: Optional["AssetCheckExecutionResolvedStatus"]
595+
) -> EntitySubset[AssetCheckKey]:
596+
"""Get subset of partitioned asset check matching specific status."""
597+
from dagster._core.storage.asset_check_partition_cache import (
598+
get_asset_check_partition_status,
599+
)
600+
601+
check_node = self.asset_graph.get(key)
602+
if not check_node or not check_node.partitions_def:
603+
check.failed(f"Asset check {key} not found or not partitioned.")
604+
605+
# Get partition status from our cache service
606+
partition_status = get_asset_check_partition_status(
607+
self._queryer.instance, key, check_node.partitions_def
608+
)
609+
610+
matching_subset = (
611+
partition_status.get_subset_for_status(status) if status else partition_status.missing
612+
)
613+
if not matching_subset.is_empty:
614+
return EntitySubset(self, key=key, value=_ValidatedEntitySubsetValue(matching_subset))
615+
else:
616+
return self.get_empty_subset(key=key)
617+
612618
async def _compute_run_in_progress_asset_subset(self, key: AssetKey) -> EntitySubset[AssetKey]:
613619
from dagster._core.storage.partition_status_cache import AssetStatusCacheValue
614620

python_modules/dagster/dagster/_core/asset_graph_view/entity_subset.py

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,13 @@ def convert_to_serializable_subset(self) -> SerializableEntitySubset[T_EntityKey
6969
return SerializableEntitySubset(key=self._key, value=self._value)
7070

7171
def expensively_compute_partition_keys(self) -> AbstractSet[str]:
72-
return {
73-
check.not_none(akpk.partition_key, "No None partition keys")
74-
for akpk in self.expensively_compute_asset_partitions()
75-
}
72+
internal_value = self.get_internal_value()
73+
if isinstance(internal_value, PartitionsSubset):
74+
return set(internal_value.get_partition_keys())
75+
elif internal_value:
76+
check.failed("No None partition keys")
77+
else:
78+
return set()
7679

7780
def expensively_compute_asset_partitions(self) -> AbstractSet[AssetKeyPartitionKey]:
7881
if not isinstance(self.key, AssetKey):
@@ -106,13 +109,25 @@ def compute_intersection(self, other: Self) -> Self:
106109
return self._oper(other, operator.and_)
107110

108111
def compute_intersection_with_partition_keys(
109-
self: "EntitySubset[AssetKey]", partition_keys: AbstractSet[str]
110-
) -> "EntitySubset[AssetKey]":
111-
key = check.inst(self.key, AssetKey)
112-
partition_subset = self._asset_graph_view.get_asset_subset_from_asset_partitions(
113-
self.key, {AssetKeyPartitionKey(key, pk) for pk in partition_keys}
112+
self: "EntitySubset[T_EntityKey]", partition_keys: AbstractSet[str]
113+
) -> "EntitySubset[T_EntityKey]":
114+
# Get partitions definition for the entity
115+
if not self.partitions_def:
116+
# Non-partitioned entity - return self if partition keys provided, else empty
117+
return self if partition_keys else self._asset_graph_view.get_empty_subset(key=self.key)
118+
119+
# Create partition subset directly from partition keys
120+
partition_subset = self.partitions_def.subset_with_partition_keys(partition_keys)
121+
122+
# Create EntitySubset with the partition subset
123+
entity_subset_with_partitions = EntitySubset(
124+
self._asset_graph_view,
125+
key=self.key,
126+
value=_ValidatedEntitySubsetValue(inner=partition_subset),
114127
)
115-
return self.compute_intersection(partition_subset)
128+
129+
# Return intersection of this subset with the partition-based subset
130+
return self.compute_intersection(entity_subset_with_partitions)
116131

117132
@cached_method
118133
def compute_parent_subset(self, parent_key: AssetKey) -> "EntitySubset[AssetKey]":

python_modules/dagster/dagster/_core/definitions/assets/graph/asset_graph.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,7 @@ def __init__(
190190
v.get_spec_for_check_key(k).description,
191191
v.get_spec_for_check_key(k).automation_condition,
192192
v.get_spec_for_check_key(k).metadata,
193+
v.get_spec_for_check_key(k).partitions_def,
193194
)
194195
for k, v in assets_defs_by_check_key.items()
195196
}

python_modules/dagster/dagster/_core/definitions/assets/graph/base_asset_graph.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -217,13 +217,15 @@ def __init__(
217217
description: Optional[str],
218218
automation_condition: Optional["AutomationCondition[AssetCheckKey]"],
219219
metadata: ArbitraryMetadataMapping,
220+
partitions_def: Optional[PartitionsDefinition],
220221
):
221222
self.key = key
222223
self.blocking = blocking
223224
self._automation_condition = automation_condition
224225
self._additional_deps = additional_deps
225226
self._description = description
226227
self._metadata = metadata
228+
self._partitions_def = partitions_def
227229

228230
@property
229231
def parent_entity_keys(self) -> AbstractSet[AssetKey]:
@@ -235,8 +237,7 @@ def child_entity_keys(self) -> AbstractSet[EntityKey]:
235237

236238
@property
237239
def partitions_def(self) -> Optional[PartitionsDefinition]:
238-
# all checks are unpartitioned
239-
return None
240+
return self._partitions_def
240241

241242
@property
242243
def partition_mappings(self) -> Mapping[EntityKey, PartitionMapping]:

python_modules/dagster/dagster/_core/definitions/assets/graph/remote_asset_graph.py

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,9 @@
4040
from dagster._core.definitions.metadata import ArbitraryMetadataMapping
4141
from dagster._core.definitions.partitions.definition import PartitionsDefinition
4242
from dagster._core.definitions.partitions.mapping import PartitionMapping
43-
from dagster._core.definitions.selector import ScheduleSelector, SensorSelector
4443
from dagster._core.definitions.utils import DEFAULT_GROUP_NAME
4544
from dagster._core.remote_representation.external import RemoteRepository
46-
from dagster._core.remote_representation.handle import RepositoryHandle
45+
from dagster._core.remote_representation.handle import InstigatorHandle, RepositoryHandle
4746
from dagster._core.workspace.workspace import CurrentWorkspace
4847
from dagster._record import ImportFrom, record
4948
from dagster._utils.cached_method import cached_method
@@ -396,33 +395,31 @@ def resolve_to_singular_repo_scoped_node(self) -> "RemoteRepositoryAssetNode":
396395
)
397396
)
398397

399-
def get_targeting_schedule_selectors(
398+
def get_targeting_schedule_handles(
400399
self,
401-
) -> Sequence[ScheduleSelector]:
400+
) -> Sequence[InstigatorHandle]:
402401
selectors = []
403402
for node in self.repo_scoped_asset_infos:
404403
for schedule_name in node.targeting_schedule_names:
405404
selectors.append(
406-
ScheduleSelector(
407-
location_name=node.handle.location_name,
408-
repository_name=node.handle.repository_name,
409-
schedule_name=schedule_name,
405+
InstigatorHandle(
406+
repository_handle=node.handle,
407+
instigator_name=schedule_name,
410408
)
411409
)
412410

413411
return selectors
414412

415-
def get_targeting_sensor_selectors(
413+
def get_targeting_sensor_handles(
416414
self,
417-
) -> Sequence[SensorSelector]:
415+
) -> Sequence[InstigatorHandle]:
418416
selectors = []
419417
for node in self.repo_scoped_asset_infos:
420418
for sensor_name in node.targeting_sensor_names:
421419
selectors.append(
422-
SensorSelector(
423-
location_name=node.handle.location_name,
424-
repository_name=node.handle.repository_name,
425-
sensor_name=sensor_name,
420+
InstigatorHandle(
421+
repository_handle=node.handle,
422+
instigator_name=sensor_name,
426423
)
427424
)
428425
return selectors
@@ -480,6 +477,9 @@ def _get_asset_check_node_from_remote_asset_check_node(
480477
remote_node.asset_check.description,
481478
remote_node.asset_check.automation_condition,
482479
{}, # metadata not yet on AssetCheckNodeSnap
480+
remote_node.asset_check.partitions_def_snapshot.get_partitions_definition()
481+
if remote_node.asset_check.partitions_def_snapshot
482+
else None,
483483
)
484484

485485
##### COMMON ASSET GRAPH INTERFACE

python_modules/dagster/dagster_tests/storage_tests/test_asset_checks.py

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,97 @@ def test_partitioned_asset_check_graph_structure():
8383
# Test: check is linked to asset
8484
asset_node = asset_graph.get(partitioned_asset.key)
8585
assert partitioned_asset_check.check_key in asset_node.check_keys
86+
87+
88+
@pytest.mark.asyncio
89+
async def test_partitioned_asset_check_subset_computation_empty():
90+
"""Test subset computation for partitioned asset checks before any executions."""
91+
from dagster._core.asset_graph_view.asset_graph_view import AssetGraphView
92+
from dagster._core.storage.asset_check_execution_record import AssetCheckExecutionResolvedStatus
93+
94+
with dg.instance_for_test() as instance:
95+
view = AssetGraphView.for_test(partitioned_defs, instance=instance)
96+
partitioned_key = partitioned_asset_check.check_key
97+
98+
# Before executions, all partitions should be missing
99+
missing_subset = await view.compute_subset_with_status(partitioned_key, None)
100+
assert not missing_subset.is_empty
101+
assert missing_subset.expensively_compute_partition_keys() == {"a", "b", "c"}
102+
103+
# No partitions should have execution statuses
104+
for status in [
105+
AssetCheckExecutionResolvedStatus.SUCCEEDED,
106+
AssetCheckExecutionResolvedStatus.FAILED,
107+
AssetCheckExecutionResolvedStatus.IN_PROGRESS,
108+
]:
109+
subset = await view.compute_subset_with_status(partitioned_key, status)
110+
assert subset.is_empty
111+
112+
113+
@pytest.mark.asyncio
114+
async def test_partitioned_asset_check_subset_computation_after_execution():
115+
"""Test subset computation after executing partitions with different outcomes."""
116+
from dagster._core.asset_graph_view.asset_graph_view import AssetGraphView
117+
from dagster._core.storage.asset_check_execution_record import AssetCheckExecutionResolvedStatus
118+
119+
with dg.instance_for_test() as instance:
120+
# Execute partitions with known outcomes
121+
result_a = dg.materialize(
122+
[partitioned_asset, partitioned_asset_check], instance=instance, partition_key="a"
123+
)
124+
assert result_a.success
125+
126+
result_b = dg.materialize(
127+
[partitioned_asset, partitioned_asset_check], instance=instance, partition_key="b"
128+
)
129+
assert result_b.success # Run succeeds but check fails
130+
131+
view = AssetGraphView.for_test(partitioned_defs, instance=instance)
132+
partitioned_key = partitioned_asset_check.check_key
133+
134+
# Test: subsets reflect execution outcomes
135+
succeeded_subset = await view.compute_subset_with_status(
136+
partitioned_key, AssetCheckExecutionResolvedStatus.SUCCEEDED
137+
)
138+
assert not succeeded_subset.is_empty
139+
assert succeeded_subset.expensively_compute_partition_keys() == {"a"}
140+
141+
failed_subset = await view.compute_subset_with_status(
142+
partitioned_key, AssetCheckExecutionResolvedStatus.FAILED
143+
)
144+
assert not failed_subset.is_empty # Should contain 'b'
145+
assert failed_subset.expensively_compute_partition_keys() == {"b"}
146+
147+
missing_subset = await view.compute_subset_with_status(partitioned_key, None)
148+
assert not missing_subset.is_empty # Should contain 'c'
149+
assert missing_subset.expensively_compute_partition_keys() == {"c"}
150+
151+
152+
@pytest.mark.asyncio
153+
async def test_non_partitioned_asset_check_compatibility():
154+
"""Test that non-partitioned asset checks still work with existing logic."""
155+
from dagster._core.asset_graph_view.asset_graph_view import AssetGraphView
156+
from dagster._core.storage.asset_check_execution_record import AssetCheckExecutionResolvedStatus
157+
158+
with dg.instance_for_test() as instance:
159+
view = AssetGraphView.for_test(defs, instance=instance)
160+
non_partitioned_key = the_asset_check.check_key
161+
162+
missing_subset = await view.compute_subset_with_status(non_partitioned_key, None)
163+
assert not missing_subset.is_partitioned
164+
165+
succeeded_subset = await view.compute_subset_with_status(
166+
non_partitioned_key, AssetCheckExecutionResolvedStatus.SUCCEEDED
167+
)
168+
assert not succeeded_subset.is_partitioned
169+
170+
# Execute the check
171+
result = dg.materialize([the_asset, the_asset_check], instance=instance)
172+
assert result.success
173+
174+
# After execution
175+
view_after = AssetGraphView.for_test(defs, instance=instance)
176+
succeeded_subset_after = await view_after.compute_subset_with_status(
177+
non_partitioned_key, AssetCheckExecutionResolvedStatus.SUCCEEDED
178+
)
179+
assert not succeeded_subset_after.is_partitioned

0 commit comments

Comments
 (0)