Skip to content

Commit 7a1d41d

Browse files
benpankowbenpankow
committed
snip
1 parent c84f92c commit 7a1d41d

19 files changed

+479
-132
lines changed

Diff for: ‎examples/docs_snippets/docs_snippets/guides/dg/using-env/1-jaffle-clone.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
git clone --depth=1 https://github.com/dagster-io/jaffle-platform.git dbt && rm -rf dbt/.git

Diff for: ‎examples/docs_snippets/docs_snippets/guides/dg/using-env/10-dg-env-list.txt

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
dg env list
2+
3+
DagsterPlusCliConfig(url=None, organization=None, default_deployment=None, user_token=None, agent_timeout=None)
4+
┏━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓
5+
┃ Env Var ┃ Local Value ┃ Components ┃
6+
┡━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━┩
7+
│ DBT_SCHEMA │ jaffle_shop │ jdbt │
8+
└────────────┴─────────────┴────────────┘

Diff for: ‎examples/docs_snippets/docs_snippets/guides/dg/using-env/11-dg-env-list.txt

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
dg env list
2+
3+
DagsterPlusCliConfig(url='http://localhost:60818', organization='hooli', default_deployment='test', user_token='test', agent_timeout=None)
4+
┏━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━┳━━━━━━━━┳━━━━━━┓
5+
┃ Env Var ┃ Local Value ┃ Components ┃ Dev ┃ Branch ┃ Full ┃
6+
┡━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━╇━━━━━━━━╇━━━━━━┩
7+
│ DBT_SCHEMA │ jaffle_shop │ jdbt │ X │ X │ │
8+
└────────────┴─────────────┴────────────┴─────┴────────┴──────┘

Diff for: ‎examples/docs_snippets/docs_snippets/guides/dg/using-env/12-dg-env-list.txt

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
dg env list
2+
3+
DagsterPlusCliConfig(url=None, organization=None, default_deployment=None, user_token=None, agent_timeout=None)
4+
┏━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━┓
5+
┃ Env Var ┃ Local Value ┃ Components ┃
6+
┡━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━┩
7+
│ DBT_SCHEMA │ jaffle_shop │ jdbt │
8+
└────────────┴─────────────┴────────────┘

Diff for: ‎examples/docs_snippets/docs_snippets/guides/dg/using-env/2-dg-list-component-types.txt

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
dg list component-type
2+
3+
Using /.../jaffle-platform/.venv/bin/dagster-components
4+
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┓
5+
┃ Component Type ┃ Summary ┃
6+
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━┩
7+
│ dagster_components.dagster.DefinitionsComponent │ Wraps an │
8+
│ │ arbitrary set of │
9+
│ │ Dagster │
10+
│ │ definitions. │
11+
│ dagster_components.dagster.DefsFolderComponent │ Wraps a │
12+
│ │ DefsModule to │
13+
│ │ allow the │
14+
│ │ addition of │
15+
│ │ arbitrary │
16+
│ │ attributes. │
17+
│ dagster_components.dagster.PipesSubprocessScriptCollectionComponent │ Assets that wrap │
18+
│ │ Python scripts │
19+
│ │ executed with │
20+
│ │ Dagster's │
21+
│ │ PipesSubprocess… │
22+
│ dagster_components.dagster_dbt.DbtProjectComponent │ Expose a DBT │
23+
│ │ project to │
24+
│ │ Dagster as a set │
25+
│ │ of assets. │
26+
└─────────────────────────────────────────────────────────────────────┴──────────────────┘

Diff for: ‎examples/docs_snippets/docs_snippets/guides/dg/using-env/3-dg-scaffold-jdbt.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
dg scaffold dagster_components.dagster_dbt.DbtProjectComponent jdbt --project-path dbt/jdbt
2+
3+
Using /.../jaffle-platform/.venv/bin/dagster-components

Diff for: ‎examples/docs_snippets/docs_snippets/guides/dg/using-env/4-dg-component-check.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
dg check yaml
2+
3+
All components validated successfully.

Diff for: ‎examples/docs_snippets/docs_snippets/guides/dg/using-env/5-project-jdbt.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
type: dagster_components.dagster_dbt.DbtProjectComponent
2+
3+
attributes:
4+
dbt:
5+
project_dir: ../../../dbt/jdbt
6+
asset_attributes:
7+
key: "jaffle_platform/{{ env('DBT_SCHEMA') }}/{{ node.name }}"

Diff for: ‎examples/docs_snippets/docs_snippets/guides/dg/using-env/6-dg-component-check.txt

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
dg check yaml
2+
3+
/.../jaffle-platform/jaffle_platform/defs/jdbt/component.yaml:1 - requires.env Component uses environment variables that are not specified in the component file: DBT_SCHEMA
4+
To automatically add these environment variables to the component file, run `dg check yaml --fix-env-requirements`
5+
|
6+
1 | type: dagster_components.dagster_dbt.DbtProjectComponent
7+
| ^ Component uses environment variables that are not specified in the component file: DBT_SCHEMA
8+
To automatically add these environment variables to the component file, run `dg check yaml --fix-env-requirements`
9+
2 |
10+
3 | attributes:
11+
4 | dbt:
12+
5 | project_dir: ../../../dbt/jdbt
13+
6 | asset_attributes:
14+
7 | key: "jaffle_platform/{{ env('DBT_SCHEMA') }}/{{ node.name }}"
15+
|

Diff for: ‎examples/docs_snippets/docs_snippets/guides/dg/using-env/7-dg-component-check.txt

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
dg check yaml --fix-env-requirements
2+
3+
Updated /.../jaffle-platform/jaffle_platform/defs/jdbt/component.yaml
4+
1 component files were updated to fix environment variable requirements.
5+
The following environment variables are used in components but not specified in the .env file or the current shell environment:
6+
DBT_SCHEMA

Diff for: ‎examples/docs_snippets/docs_snippets/guides/dg/using-env/8-dg-component-check.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
echo 'DBT_SCHEMA=jaffle_shop' >> .env

Diff for: ‎examples/docs_snippets/docs_snippets/guides/dg/using-env/9-dg-component-check.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
dg check yaml
2+
3+
All components validated successfully.

Diff for: ‎examples/docs_snippets/docs_snippets_tests/snippet_checks/guides/dg/test_using_env.py

+242
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
import os
2+
from http.server import HTTPServer
3+
from pathlib import Path
4+
from typing import Any, Callable, Optional
5+
6+
import responses
7+
from dagster_dg.utils import ensure_dagster_dg_tests_import
8+
from dagster_dg.utils.plus import gql
9+
from pytest_httpserver import HTTPServer
10+
from werkzeug import Request, Response
11+
12+
from dagster._utils.env import environ
13+
from docs_snippets_tests.snippet_checks.guides.components.utils import (
14+
DAGSTER_ROOT,
15+
EDITABLE_DIR,
16+
MASK_JAFFLE_PLATFORM,
17+
format_multiline,
18+
isolated_snippet_generation_environment,
19+
)
20+
from docs_snippets_tests.snippet_checks.utils import (
21+
_run_command,
22+
check_file,
23+
compare_tree_output,
24+
create_file,
25+
run_command_and_snippet_output,
26+
)
27+
28+
ensure_dagster_dg_tests_import()
29+
from dagster_dg_tests.cli_tests.plus_tests.utils import mock_gql_response, responses
30+
31+
MASK_VENV = (r"Using.*\.venv.*", "")
32+
33+
34+
SNIPPETS_DIR = (
35+
DAGSTER_ROOT
36+
/ "examples"
37+
/ "docs_snippets"
38+
/ "docs_snippets"
39+
/ "guides"
40+
/ "dg"
41+
/ "using-env"
42+
)
43+
import json
44+
45+
gql_matchers: list[tuple[Callable[[Request], bool], dict[str, Any]]] = []
46+
47+
48+
def mock_gql_mutation(
49+
mutation: str,
50+
json_data: dict[str, Any],
51+
expected_variables: Optional[dict[str, Any]] = None,
52+
) -> None:
53+
def match(request: Request) -> bool:
54+
json_body = request.json or {}
55+
body_query_first_line_normalized = (
56+
json_body["query"].strip().split("\n")[0].strip()
57+
)
58+
query_first_line_normalized = mutation.strip().split("\n")[0].strip()
59+
if expected_variables and json_body["variables"] != expected_variables:
60+
return False
61+
return body_query_first_line_normalized == query_first_line_normalized
62+
63+
gql_matchers.append((match, json_data))
64+
65+
66+
def mock_gql_response(
67+
query: str,
68+
json_data: dict[str, Any],
69+
) -> None:
70+
def match(request: Request) -> bool:
71+
json_body = request.json or {}
72+
print(json_body)
73+
body_query_first_line_normalized = (
74+
json_body["query"].strip().split("\n")[0].strip()
75+
)
76+
query_first_line_normalized = query.strip().split("\n")[0].strip()
77+
print(body_query_first_line_normalized, query_first_line_normalized)
78+
return True
79+
return body_query_first_line_normalized == query_first_line_normalized
80+
81+
gql_matchers.append((match, json_data))
82+
83+
84+
@responses.activate
85+
def test_component_docs_using_env(
86+
update_snippets: bool, httpserver: HTTPServer
87+
) -> None:
88+
with isolated_snippet_generation_environment() as get_next_snip_number:
89+
_run_command(
90+
cmd="dg scaffold project jaffle-platform --use-editable-dagster && cd jaffle-platform",
91+
)
92+
_run_command(cmd="uv venv")
93+
_run_command(cmd="uv sync")
94+
_run_command(
95+
f"uv add --editable '{EDITABLE_DIR / 'dagster-components'!s}' '{DAGSTER_ROOT / 'python_modules' / 'dagster'!s}' '{DAGSTER_ROOT / 'python_modules' / 'dagster-webserver'!s}'"
96+
)
97+
98+
# Set up dbt
99+
run_command_and_snippet_output(
100+
cmd="git clone --depth=1 https://github.com/dagster-io/jaffle-platform.git dbt && rm -rf dbt/.git",
101+
snippet_path=SNIPPETS_DIR / f"{get_next_snip_number()}-jaffle-clone.txt",
102+
update_snippets=update_snippets,
103+
ignore_output=True,
104+
)
105+
_run_command(
106+
f"uv add --editable '{EDITABLE_DIR / 'dagster-dbt'!s}' && uv add --editable '{EDITABLE_DIR / 'dagster-components'!s}[dbt]'; uv add dbt-duckdb"
107+
)
108+
run_command_and_snippet_output(
109+
cmd="dg list component-type",
110+
snippet_path=SNIPPETS_DIR
111+
/ f"{get_next_snip_number()}-dg-list-component-types.txt",
112+
update_snippets=update_snippets,
113+
snippet_replace_regex=[MASK_JAFFLE_PLATFORM],
114+
)
115+
116+
# Scaffold dbt project components
117+
run_command_and_snippet_output(
118+
cmd="dg scaffold dagster_components.dagster_dbt.DbtProjectComponent jdbt --project-path dbt/jdbt",
119+
snippet_path=SNIPPETS_DIR
120+
/ f"{get_next_snip_number()}-dg-scaffold-jdbt.txt",
121+
update_snippets=update_snippets,
122+
snippet_replace_regex=[MASK_JAFFLE_PLATFORM],
123+
)
124+
run_command_and_snippet_output(
125+
cmd="dg check yaml",
126+
snippet_path=SNIPPETS_DIR
127+
/ f"{get_next_snip_number()}-dg-component-check.txt",
128+
update_snippets=update_snippets,
129+
snippet_replace_regex=[
130+
MASK_JAFFLE_PLATFORM,
131+
],
132+
)
133+
create_file(
134+
Path("jaffle_platform") / "defs" / "jdbt" / "component.yaml",
135+
snippet_path=SNIPPETS_DIR / f"{get_next_snip_number()}-project-jdbt.yaml",
136+
contents=format_multiline("""
137+
type: dagster_components.dagster_dbt.DbtProjectComponent
138+
139+
attributes:
140+
dbt:
141+
project_dir: ../../../dbt/jdbt
142+
asset_attributes:
143+
key: "jaffle_platform/{{ env('DBT_SCHEMA') }}/{{ node.name }}"
144+
"""),
145+
)
146+
run_command_and_snippet_output(
147+
cmd="dg check yaml",
148+
snippet_path=SNIPPETS_DIR
149+
/ f"{get_next_snip_number()}-dg-component-check.txt",
150+
update_snippets=update_snippets,
151+
snippet_replace_regex=[
152+
MASK_JAFFLE_PLATFORM,
153+
],
154+
expect_error=True,
155+
)
156+
run_command_and_snippet_output(
157+
cmd="dg check yaml --fix-env-requirements",
158+
snippet_path=SNIPPETS_DIR
159+
/ f"{get_next_snip_number()}-dg-component-check.txt",
160+
update_snippets=update_snippets,
161+
snippet_replace_regex=[
162+
MASK_JAFFLE_PLATFORM,
163+
],
164+
expect_error=True,
165+
)
166+
run_command_and_snippet_output(
167+
cmd="echo 'DBT_SCHEMA=jaffle_shop' >> .env",
168+
snippet_path=SNIPPETS_DIR
169+
/ f"{get_next_snip_number()}-dg-component-check.txt",
170+
update_snippets=update_snippets,
171+
snippet_replace_regex=[
172+
MASK_JAFFLE_PLATFORM,
173+
],
174+
)
175+
run_command_and_snippet_output(
176+
cmd="dg check yaml",
177+
snippet_path=SNIPPETS_DIR
178+
/ f"{get_next_snip_number()}-dg-component-check.txt",
179+
update_snippets=update_snippets,
180+
)
181+
run_command_and_snippet_output(
182+
cmd="dg env list",
183+
snippet_path=SNIPPETS_DIR / f"{get_next_snip_number()}-dg-env-list.txt",
184+
update_snippets=update_snippets,
185+
)
186+
187+
mock_gql_mutation(
188+
gql.GET_SECRETS_FOR_SCOPES_QUERY,
189+
json_data={
190+
"data": {
191+
"secretsForScopes": {
192+
"secrets": [
193+
{
194+
"secretName": "DBT_SCHEMA",
195+
"locationNames": ["jaffle-platform"],
196+
"fullDeploymentScope": False,
197+
"allBranchDeploymentsScope": True,
198+
"localDeploymentScope": True,
199+
}
200+
]
201+
}
202+
}
203+
},
204+
expected_variables={
205+
"locationName": "jaffle-platform",
206+
"scopes": {
207+
"fullDeploymentScope": True,
208+
"allBranchDeploymentsScope": True,
209+
"localDeploymentScope": True,
210+
},
211+
"secretName": "DBT_SCHEMA",
212+
},
213+
)
214+
215+
def _handle(request: Request) -> Response:
216+
for match, data in gql_matchers:
217+
if match(request):
218+
return Response(json.dumps(data), status=200)
219+
return Response(
220+
json.dumps({}),
221+
status=200,
222+
)
223+
224+
httpserver.expect_request("/hooli/graphql").respond_with_handler(_handle)
225+
226+
Path(os.environ["DG_CLI_CONFIG"]).write_text(
227+
f"""
228+
[cli.telemetry]
229+
enabled = false
230+
[cli.plus]
231+
organization = "hooli"
232+
url = "{httpserver.url_for('').rstrip('/')}"
233+
user_token = "test"
234+
default_deployment = "test"
235+
"""
236+
)
237+
238+
run_command_and_snippet_output(
239+
cmd="dg env list",
240+
snippet_path=SNIPPETS_DIR / f"{get_next_snip_number()}-dg-env-list.txt",
241+
update_snippets=update_snippets,
242+
)

Diff for: ‎examples/docs_snippets/docs_snippets_tests/snippet_checks/utils.py

+11-3
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,11 @@ def re_ignore_after(match_str: str) -> tuple[str, str]:
5252
PWD_REGEX = re.compile(r"PWD=(.*?);")
5353

5454

55-
def _run_command(cmd: Union[str, Sequence[str]], expect_error: bool = False) -> str:
55+
def _run_command(
56+
cmd: Union[str, Sequence[str]],
57+
expect_error: bool = False,
58+
stdin: Optional[str] = None,
59+
) -> str:
5660
if not isinstance(cmd, str):
5761
cmd = " ".join(cmd)
5862

@@ -62,7 +66,10 @@ def _run_command(cmd: Union[str, Sequence[str]], expect_error: bool = False) ->
6266
else:
6367
actual_output = (
6468
subprocess.check_output(
65-
f'{cmd} && echo "PWD=$(pwd);"', shell=True, stderr=subprocess.STDOUT
69+
f'{cmd} && echo "PWD=$(pwd);"',
70+
shell=True,
71+
stderr=subprocess.STDOUT,
72+
input=stdin.encode("utf-8") if stdin else None,
6673
)
6774
.decode("utf-8")
6875
.strip()
@@ -293,6 +300,7 @@ def run_command_and_snippet_output(
293300
ignore_output: bool = False,
294301
expect_error: bool = False,
295302
print_cmd: Optional[str] = None,
303+
stdin: Optional[str] = None,
296304
):
297305
"""Run the given command and check that the output matches the contents of the snippet
298306
at `snippet_path`. If `update_snippets` is `True`, updates the snippet file with the
@@ -315,7 +323,7 @@ def run_command_and_snippet_output(
315323
"""
316324
assert update_snippets is not None or snippet_path is None
317325

318-
output = _run_command(cmd, expect_error=expect_error)
326+
output = _run_command(cmd, expect_error=expect_error, stdin=stdin)
319327

320328
if snippet_path:
321329
assert update_snippets is not None

Diff for: ‎examples/docs_snippets/setup.py

+1
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
"slack_sdk",
4040
"syrupy",
4141
"xgboost",
42+
"pytest-httpserver",
4243
],
4344
},
4445
)

0 commit comments

Comments
 (0)