Skip to content

Commit c7f8003

Browse files
neverettneverett
committed
Merge branch 'master' into nikki/docs/combine-integrations-docs
2 parents ba557d7 + d7791c5 commit c7f8003

107 files changed

Lines changed: 10112 additions & 2915 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎CHANGES.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
- Optimized performance of calculating partition keys for time window partitions with exclusions.
88
- `timedelta` and `datetime` are now available via the `datetime` context when rendering components (Thanks, [@stevenayers](https://github.com/stevenayers)!)
9-
- `FreshnessPolicy` is now available via the `dg` context when rendering components. (Thanks, [@stevenayers](@https://github.com/stevenayers)!)
9+
- `FreshnessPolicy` is now available via the `dg` context when rendering components. (Thanks, [@stevenayers](https://github.com/stevenayers)!)
1010
- Assets may now be annotated with up to 10 kinds (limit was previously 3).
1111
- Arbitrary resource parameters may now be hidden in the UI by setting `json_schema_extra={"dagster__is_secret": True}` on the corresponding `Field` definition in the resource class.
1212
- The `dg docs` cli group has been removed. The `integrations` subcommand has been moved to `dg utils integrations`.

‎docs/docs/deployment/dagster-plus/management/dagster-ips.md‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,10 @@ The Dagster+ web interface, CLI, and GraphQL API use [AWS Cloudfront's content d
1212

1313
The Dagster+ agent interacts with the following IP addresses:
1414

15-
```
15+
<Tabs groupId="region">
16+
<TabItem value="us-region" label="US region">
17+
18+
```plain
1619
34.215.239.157
1720
35.165.239.109
1821
35.83.161.124
@@ -23,15 +26,30 @@ The Dagster+ agent interacts with the following IP addresses:
2326
54.188.126.120
2427
```
2528

29+
</TabItem>
30+
<TabItem value="eu-region" label="EU region">
31+
32+
```plain
33+
13.50.180.120
34+
13.51.19.57
35+
13.60.164.107
36+
13.62.96.222
37+
13.62.107.201
38+
51.21.15.208
39+
```
40+
41+
</TabItem>
42+
</Tabs>
43+
2644
:::note
27-
Additional IP addresses may be added over time. This list was last updated on **February 7, 2025**.
45+
Additional IP addresses may be added over time. This list was last updated on **December 17, 2025**.
2846
:::
2947

3048
## URLs
3149

3250
In addition to these IP addresses, the following URLs also need to be allowed egress access from your agent:
3351

34-
<Tabs>
52+
<Tabs groupId="region">
3553
<TabItem value="us-region" label="US region">
3654
- `cloud-prod-object-snapshots.s3.amazonaws.com`
3755
- `cloud-prod-compute-logs.s3.amazonaws.com` - Only required if compute logs are being sent to Dagster+'s control plane. Not needed if you've configured compute logs to be sent to your own blob storage. See [Managing compute logs and error messages](/deployment/dagster-plus/management/managing-compute-logs-and-error-messages) for more details.

‎docs/docs/deployment/dagster-plus/serverless/dagster-ips.md‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ tags: [dagster-plus-feature]
88

99
Dagster+ Serverless code will make requests from one of the following IP addresses. You may need to whitelist / allowlist them for services your code interacts with.
1010

11+
<Tabs>
12+
<TabItem value="us-region" label="US region">
13+
1114
```plain
1215
34.216.9.66
1316
35.162.181.243
@@ -26,8 +29,20 @@ Dagster+ Serverless code will make requests from one of the following IP address
2629
54.71.18.84
2730
```
2831

32+
</TabItem>
33+
<TabItem value="eu-region" label="EU region">
34+
35+
```plain
36+
16.170.24.133
37+
13.62.119.65
38+
56.228.54.244
39+
```
40+
41+
</TabItem>
42+
</Tabs>
43+
2944
:::note
3045

31-
Additional IP addresses may be added over time. This list was last updated on **October 24, 2024**.
46+
Additional IP addresses may be added over time. This list was last updated on **December 17, 2025**.
3247

3348
:::

‎docs/docusaurus.config.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ const config: Config = {
224224
lastVersion: 'current',
225225
versions: {
226226
current: {
227-
label: 'Latest (1.12.6)',
227+
label: 'Latest (1.12.7)',
228228
path: '/',
229229
},
230230
},

‎docs/tox.ini‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ deps =
2626
# Can't stub deps because processed by sphinx-click
2727
-e ../python_modules/dagster
2828
-e ../python_modules/libraries/dagster-dg-core
29+
-e ../python_modules/libraries/dagster-cloud-cli
2930
-e ../python_modules/dagster-pipes
3031
-e ../python_modules/dagster-graphql
3132
-e ../python_modules/dagster-webserver
@@ -38,6 +39,7 @@ deps =
3839
-e ../python_modules/libraries/dagster-datahub
3940
-e ../python_modules/libraries/dagster-gcp[dataproc]
4041
-e ../python_modules/libraries/dagster-pyspark
42+
-e ../python_modules/libraries/dagster-spark
4143
-e ../python_modules/libraries/dagster-ssh
4244
-e ../python_modules/libraries/dagster-duckdb
4345
-e ../python_modules/libraries/dagster-dbt
@@ -52,6 +54,7 @@ deps =
5254
-e ../python_modules/libraries/dagster-sigma
5355
-e ../python_modules/libraries/dagster-tableau
5456
-e ../python_modules/libraries/dagster-powerbi
57+
-e ../python_modules/libraries/dagster-pandas
5558
-e ../python_modules/libraries/dagster-airlift[tutorial,core,in-airflow]
5659
-e ../examples/airlift-migration-tutorial
5760
sling
@@ -70,6 +73,7 @@ deps =
7073
# Since M3 Macs and Windows systems cannot parallelize the build,
7174
# do not set -j flag
7275
commands =
76+
!windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster | grep -v dagster-iceberg | grep -v dagster-polars'
7377
make --directory=sphinx clean
7478
make --directory=sphinx mdx SPHINXOPTS="-W"
7579

‎examples/project_databricks_and_snowflake/.env.example‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Databricks Configuration
2+
DATABRICKS_SERVER_HOSTNAME=your-workspace.cloud.databricks.com
3+
DATABRICKS_HTTP_PATH=/sql/1.0/warehouses/your-warehouse-id
4+
DATABRICKS_TOKEN=dapi_your_token_here
5+
DELTA_STORAGE_PATH=data/delta
6+
7+
# Snowflake Configuration
8+
SNOWFLAKE_ACCOUNT=your-account.region
9+
SNOWFLAKE_USER=your_user
10+
SNOWFLAKE_PASSWORD=your_password
11+
SNOWFLAKE_WAREHOUSE=COMPUTE_WH
12+
SNOWFLAKE_DATABASE=your_database
13+
SNOWFLAKE_SCHEMA=PUBLIC
14+
15+
# Demo Mode (set to empty string to disable)
16+
DEMO_MODE=true

‎examples/project_databricks_and_snowflake/README.md‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
# Databricks and Snowflake Example
2+
3+
A Dagster workspace demonstrating how to build heterogeneous data platforms using multiple code locations. This project showcases production data platform patterns with Databricks Delta Lake and Snowflake dbt medallion architectures as separate projects within a workspace.
4+
5+
## Project Structure
6+
7+
```
8+
project_databricks_and_snowflake/
9+
├── dg.toml # Workspace configuration
10+
├── dagster_cloud.yaml # Dagster+ deployment configuration
11+
├── deployments/
12+
│ └── local/
13+
│ └── pyproject.toml # Local environment for dg commands
14+
├── projects/
15+
│ ├── databricks-delta/ # Code location 1
16+
│ │ ├── pyproject.toml
17+
│ │ ├── tests/
18+
│ │ └── src/databricks_delta/
19+
│ │ ├── definitions.py
20+
│ │ └── defs/
21+
│ │ ├── components/lakehouse/
22+
│ │ └── resources.py
23+
│ └── snowflake-medallion/ # Code location 2
24+
│ ├── pyproject.toml
25+
│ ├── tests/
26+
│ └── src/snowflake_medallion/
27+
│ ├── definitions.py
28+
│ ├── dbt_project/
29+
│ └── defs/
30+
│ ├── components/medallion/
31+
│ └── resources.py
32+
├── .env.example
33+
├── pyproject.toml
34+
└── README.md
35+
```
36+
37+
## Getting Started
38+
39+
### Prerequisites
40+
41+
- Python 3.10+
42+
- [uv](https://github.com/astral-sh/uv) package manager
43+
44+
### Installation
45+
46+
1. Set up the local deployment environment:
47+
48+
```bash
49+
cd deployments/local
50+
uv sync
51+
source .venv/bin/activate
52+
```
53+
54+
2. Set up each project environment:
55+
56+
```bash
57+
cd ../../projects/databricks-delta
58+
uv sync
59+
60+
cd ../snowflake-medallion
61+
uv sync
62+
```
63+
64+
3. Copy the environment file and configure credentials:
65+
66+
```bash
67+
cp .env.example .env
68+
# Edit .env with your credentials
69+
```
70+
71+
### Running the Workspace
72+
73+
From the workspace root with the local environment activated:
74+
75+
```bash
76+
cd /path/to/project_databricks_and_snowflake
77+
source deployments/local/.venv/bin/activate
78+
dg dev
79+
```
80+
81+
This starts the Dagster webserver at `http://localhost:3000` and loads both code locations.
82+
83+
## Code Locations (Projects)
84+
85+
### 1. Databricks Delta
86+
87+
**Pattern**: Lakehouse with Medallion Architecture
88+
**Storage**: Delta Lake tables
89+
**Technology**: Databricks, Delta Lake
90+
91+
Implements the medallion architecture with:
92+
93+
- **Bronze Layer**: Raw sensor data loaded to Delta tables
94+
- **Silver Layer**: Cleaned and validated sensor data
95+
- **Gold Layer**: Aggregated sensor summaries
96+
97+
### 2. Snowflake Medallion
98+
99+
**Pattern**: ELT with Medallion Architecture
100+
**Storage**: Snowflake tables
101+
**Technology**: Snowflake, dbt, SQL transformations
102+
103+
Implements the medallion architecture with:
104+
105+
- **Bronze Layer**: Raw customer data loaded from sources
106+
- **Silver Layer**: Cleaned and validated customer data (dbt models)
107+
- **Gold Layer**: Curated customer summaries (dbt models)
108+
109+
## Demo Mode
110+
111+
Both projects support demo mode for local development without external dependencies:
112+
113+
- **Databricks Delta**: Uses local file system instead of Databricks
114+
- **Snowflake Medallion**: Uses DuckDB adapter instead of Snowflake
115+
116+
Demo mode is automatically enabled when environment variables are not set.
117+
118+
## Environment Variables
119+
120+
### Databricks Configuration
121+
122+
```bash
123+
DATABRICKS_SERVER_HOSTNAME=your-workspace.cloud.databricks.com
124+
DATABRICKS_HTTP_PATH=/sql/1.0/warehouses/your-warehouse-id
125+
DATABRICKS_TOKEN=dapi_your_token_here
126+
DELTA_STORAGE_PATH=data/delta
127+
```
128+
129+
### Snowflake Configuration
130+
131+
```bash
132+
SNOWFLAKE_ACCOUNT=your-account.region
133+
SNOWFLAKE_USER=your_user
134+
SNOWFLAKE_PASSWORD=your_password
135+
SNOWFLAKE_WAREHOUSE=COMPUTE_WH
136+
SNOWFLAKE_DATABASE=your_database
137+
SNOWFLAKE_SCHEMA=PUBLIC
138+
```
139+
140+
## Running Tests
141+
142+
Each project has its own tests. Run them from within each project directory:
143+
144+
```bash
145+
# Databricks Delta tests
146+
cd projects/databricks-delta
147+
source .venv/bin/activate
148+
pytest tests/ -v
149+
150+
# Snowflake Medallion tests
151+
cd projects/snowflake-medallion
152+
source .venv/bin/activate
153+
pytest tests/ -v
154+
```
155+
156+
## Deploying to Dagster+
157+
158+
This project includes a `dagster_cloud.yaml` file for deploying to Dagster+. The configuration defines both code locations with their respective environment variables and secrets.
159+
160+
### Setup
161+
162+
1. Configure environment variables in Dagster+:
163+
- Add non-sensitive variables (account names, hostnames) as environment variables
164+
- Add sensitive values (tokens, passwords) as secrets
165+
166+
2. Deploy using the Dagster+ CLI or GitHub integration:
167+
168+
```bash
169+
dagster-cloud workspace sync
170+
```
171+
172+
For more details, see the [Dagster+ deployment documentation](https://docs.dagster.io/dagster-plus/deployment/code-locations/dagster-cloud-yaml).

‎examples/project_databricks_and_snowflake/dagster_cloud.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Dagster+ deployment configuration
2+
# See: https://docs.dagster.io/dagster-plus/deployment/code-locations/dagster-cloud-yaml
3+
4+
locations:
5+
- location_name: databricks-delta
6+
code_source:
7+
package_name: databricks_delta
8+
build:
9+
directory: ./projects/databricks-delta
10+
container_context:
11+
k8s:
12+
env_vars:
13+
- DATABRICKS_SERVER_HOSTNAME
14+
- DATABRICKS_HTTP_PATH
15+
- DELTA_STORAGE_PATH
16+
env_secrets:
17+
- DATABRICKS_TOKEN
18+
19+
- location_name: snowflake-medallion
20+
code_source:
21+
package_name: snowflake_medallion
22+
build:
23+
directory: ./projects/snowflake-medallion
24+
container_context:
25+
k8s:
26+
env_vars:
27+
- SNOWFLAKE_ACCOUNT
28+
- SNOWFLAKE_USER
29+
- SNOWFLAKE_WAREHOUSE
30+
- SNOWFLAKE_DATABASE
31+
- SNOWFLAKE_SCHEMA
32+
env_secrets:
33+
- SNOWFLAKE_PASSWORD

‎examples/project_databricks_and_snowflake/deployments/local/pyproject.toml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[project]
2+
name = "multi-platform-data-local"
3+
requires-python = ">=3.10,<3.14"
4+
version = "0.1.0"
5+
dependencies = [
6+
"dagster>=1.9.0",
7+
"dagster-webserver>=1.9.0",
8+
"dagster-dg-cli>=1.0.0",
9+
]
10+
11+
[tool.hatch.build.targets.wheel]
12+
only-include = []

0 commit comments

Comments
 (0)