Skip to content

Commit 7ca1e9f

Browse files
committed
update the example to reduce ambiguity.
1 parent c55f5ee commit 7ca1e9f

1 file changed

Lines changed: 25 additions & 8 deletions

File tree

examples/docs_snippets/docs_snippets/integrations/hf_datasets/dataset_pipeline_example.py

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
1-
from dagster_hf_datasets._export import HFDatasetPublisher
2-
from dagster_hf_datasets.assets import hf_dataset_asset
3-
from dagster_hf_datasets.io_manager import HFParquetIOManager
4-
from dagster_hf_datasets.resources import HuggingFaceResource
1+
from dagster_hf_datasets import (
2+
HFDatasetPublisher,
3+
HFParquetIOManager,
4+
HuggingFaceResource,
5+
hf_dataset_asset,
6+
)
57
from datasets import Dataset
68

7-
from dagster import AssetExecutionContext, Definitions, asset
9+
from dagster import AssetExecutionContext, Definitions, MaterializeResult, asset
810

911

1012
@hf_dataset_asset(
@@ -14,11 +16,27 @@
1416
group_name="golden_glue_pipeline",
1517
io_manager_key="hf_parquet_io_manager",
1618
)
17-
def raw_glue_qqp():
19+
def raw_glue_qqp(
20+
context: AssetExecutionContext,
21+
dataset: Dataset,
22+
) -> MaterializeResult:
1823
"""Load the raw GLUE QQP training split
1924
from the Hugging Face Hub.
2025
"""
21-
pass
26+
context.log.info(
27+
"Loaded raw dataset with %s rows",
28+
len(dataset),
29+
)
30+
31+
return MaterializeResult(
32+
value=dataset,
33+
metadata={
34+
"rows": len(dataset),
35+
"source_dataset": "nyu-mll/glue",
36+
"split": "train",
37+
"config": "qqp",
38+
},
39+
)
2240

2341

2442
@asset(
@@ -91,7 +109,6 @@ def normalize(example: dict) -> dict:
91109

92110

93111
@asset(
94-
io_manager_key="hf_parquet_io_manager",
95112
group_name="golden_glue_pipeline",
96113
)
97114
def publish_golden_glue(

0 commit comments

Comments
 (0)