Skip to content

Commit 22779dd

Browse files
committed
Fix ruff
1 parent 1063939 commit 22779dd

2 files changed

Lines changed: 41 additions & 20 deletions

File tree

python_modules/libraries/dagster-dlt/dagster_dlt/components/dlt_load_collection/component.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,8 +81,14 @@ class DltLoadSpecModel(Resolvable):
8181
translation: (
8282
Annotated[
8383
TranslationFn[DltResourceTranslatorData],
84+
TranslationFnResolver[DltResourceTranslatorData](
85+
lambda data: {"resource": data.resource, "pipeline": data.pipeline}
86+
),
87+
]
88+
| None
89+
) = None
8490
partitions_def: Annotated[
85-
Optional[PartitionsDefinition],
91+
PartitionsDefinition | None,
8692
Resolver(
8793
resolve_partitions_def,
8894
model_field_type=HourlyPartitionsDefinitionModel
@@ -93,7 +99,7 @@ class DltLoadSpecModel(Resolvable):
9399
),
94100
] = None
95101
backfill_policy: Annotated[
96-
Optional[BackfillPolicy],
102+
BackfillPolicy | None,
97103
Resolver(
98104
resolve_backfill_policy,
99105
model_field_type=SingleRunBackfillPolicyModel | MultiRunBackfillPolicyModel,

python_modules/libraries/dagster-dlt/dagster_dlt_tests/test_dlt_load_collection_component.py

Lines changed: 33 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -367,26 +367,41 @@ def get_asset_spec(self, data) -> AssetSpec:
367367
}
368368

369369

370-
def test_daily_partitioned_component(dlt_pipeline: Pipeline):
371-
"""Test that partitions_def and backfill_policy are correctly applied to assets."""
372-
partitions_def = DailyPartitionsDefinition(start_date="2024-01-01")
373-
backfill_policy = BackfillPolicy.single_run()
370+
def partitioned_load():
371+
import dlt
374372

375-
context = ComponentTree.for_test().load_context
376-
defs = DltLoadCollectionComponent(
377-
loads=[
378-
DltLoadSpecModel(
379-
pipeline=dlt_pipeline,
380-
source=dlt_source(),
381-
partitions_def=partitions_def,
382-
backfill_policy=backfill_policy,
383-
)
384-
]
385-
).build_defs(context)
373+
from dagster_dlt_tests.dlt_test_sources.duckdb_with_transformer import pipeline as dlt_source
386374

387-
assets_def = defs.resolve_assets_def(AssetKey(["example", "repos"]))
388-
assert assets_def.partitions_def == partitions_def
389-
assert assets_def.backfill_policy == backfill_policy
375+
source = dlt_source()
376+
pipeline = dlt.pipeline(
377+
pipeline_name="pipeline",
378+
destination="duckdb",
379+
dataset_name="example",
380+
)
381+
382+
383+
def test_daily_partitioned_component() -> None:
384+
"""Test that partitions_def and backfill_policy from YAML are applied to assets."""
385+
with setup_dlt_component(
386+
load_py_contents=partitioned_load,
387+
defs_yaml_contents={
388+
"type": "dagster_dlt.DltLoadCollectionComponent",
389+
"attributes": {
390+
"loads": [
391+
{
392+
"pipeline": ".load.pipeline",
393+
"source": ".load.source",
394+
"partitions_def": {"type": "daily", "start_date": "2024-01-01"},
395+
"backfill_policy": {"type": "single_run"},
396+
}
397+
]
398+
},
399+
},
400+
setup_dlt_sources=lambda: None,
401+
) as (_component, defs):
402+
assets_def = defs.resolve_assets_def(AssetKey(["example", "repos"]))
403+
assert assets_def.partitions_def == DailyPartitionsDefinition(start_date="2024-01-01")
404+
assert assets_def.backfill_policy == BackfillPolicy.single_run()
390405

391406

392407
def test_no_partitions_backwards_compatible(dlt_pipeline: Pipeline):

0 commit comments

Comments
 (0)