Skip to content

Commit 4fb0017

Browse files
cnolanminichDagster Devtools
authored andcommitted
[dagster-dbt] Gracefully skip insights on unsupported adapters in with_insights (#25440)
## Summary & Motivation Matches the behavior of `.with_insights()` chained method with the older `dbt_with_snowflake_insights()` method for generating dbt snowflake insights, where calling the method on an adapter that doesn't support cost insights raises a warning instead of an error (since there are cases, like with out demo, where you want to use different database engines without changing the code). ## Test Plan BK ## Changelog - [dagster-dbt] using the `.with_insights()` method now sends a warning instead of an error if you are not using a supported adapter Internal-RevId: 51a35ce08ae3101758315258f3ff1c5badc48280
1 parent cd7ed61 commit 4fb0017

2 files changed

Lines changed: 24 additions & 11 deletions

File tree

python_modules/libraries/dagster-dbt/dagster_dbt/core/dbt_event_iterator.py

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,9 @@ def with_insights(
326326
record_observation_usage: bool = True,
327327
) -> "DbtEventIterator[DbtDagsterEventType]":
328328
"""Associate each warehouse query with the produced asset materializations for use in Dagster
329-
Plus Insights. Currently supports Snowflake and BigQuery.
329+
Plus Insights. Currently supports Snowflake and BigQuery. For any other adapter (e.g.
330+
DuckDB), this is a no-op: a warning is logged and the dbt events pass through unchanged,
331+
so the same pipeline can run locally against an unsupported warehouse without failing.
330332
331333
For more information, see the documentation for
332334
`dagster_cloud.dagster_insights.dbt_with_snowflake_insights` and
@@ -393,6 +395,12 @@ def jaffle_shop_dbt_assets(
393395
dbt_cli_invocation=self._dbt_cli_invocation,
394396
)
395397
else:
396-
check.failed(
397-
f"The `with_insights` method is only supported for Snowflake and BigQuery and is not supported for adapter type `{adapter_type}`"
398+
logger.warning(
399+
"Dagster+ Insights is only supported for the Snowflake and BigQuery dbt"
400+
f" adapters, but the dbt project uses the `{adapter_type}` adapter. Skipping"
401+
" insights; dbt events will pass through unchanged."
402+
)
403+
return DbtEventIterator(
404+
events=self,
405+
dbt_cli_invocation=self._dbt_cli_invocation,
398406
)

python_modules/libraries/dagster-dbt/dagster_dbt_tests/core/test_postprocessing.py

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
_check as check,
1111
materialize,
1212
)
13-
from dagster._check import CheckError
1413
from dagster._core.definitions.events import AssetMaterialization, Output
1514
from dagster._core.definitions.metadata.metadata_value import MetadataValue, TableMetadataValue
1615
from dagster._core.definitions.metadata.table import TableRecord
@@ -138,21 +137,27 @@ def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource):
138137
), str(metadata_by_asset_key)
139138

140139

141-
def test_insights_err_not_snowflake_or_bq(
140+
def test_insights_skips_unsupported_adapter(
142141
test_jaffle_shop_manifest_standalone_duckdb_dbfile: dict[str, Any],
143142
caplog: pytest.LogCaptureFixture,
144143
) -> None:
144+
# On an unsupported adapter (here, DuckDB), `with_insights` is a graceful no-op: it logs a
145+
# warning and passes the dbt events through unchanged rather than failing the run.
145146
@dbt_assets(manifest=test_jaffle_shop_manifest_standalone_duckdb_dbfile)
146147
def my_dbt_assets(context: AssetExecutionContext, dbt: DbtCliResource):
147148
yield from dbt.cli(["build"], context=context).stream().with_insights()
148149

149-
with pytest.raises(CheckError) as exc_info:
150-
materialize(
151-
[my_dbt_assets],
152-
resources={"dbt": DbtCliResource(project_dir=os.fspath(test_jaffle_shop_path))},
153-
)
150+
result = materialize(
151+
[my_dbt_assets],
152+
resources={"dbt": DbtCliResource(project_dir=os.fspath(test_jaffle_shop_path))},
153+
)
154154

155-
assert "is not supported for adapter type `duckdb`" in str(exc_info.value)
155+
assert result.success
156+
assert any(
157+
"Insights is only supported for the Snowflake and BigQuery dbt adapters" in record.message
158+
and "`duckdb`" in record.message
159+
for record in caplog.records
160+
), [record.message for record in caplog.records]
156161

157162

158163
@pytest.mark.parametrize(

0 commit comments

Comments
 (0)