Skip to content

Commit f02fef7

Browse files
wdbaruniwdbaruni
committed
add duckdb network back
1 parent c6cb551 commit f02fef7

File tree

6 files changed

+216
-1
lines changed

6 files changed

+216
-1
lines changed

‎distributed-datawarehouse/duckdb/README.md

+4-1
Original file line numberDiff line numberDiff line change
@@ -154,8 +154,11 @@ Each approach has its benefits:
154154
```bash
155155
# Clone and start the multi-region environment
156156
git clone https://github.com/bacalhau-project/examples.git
157-
cd examples/setting-up-bacalhau-cluster/docker-compose/multi-region
157+
cd distributed-datawarehouse/duckdb/network
158158
docker compose up -d
159+
160+
# Move to the jobs directory
161+
cd ../jobs
159162
```
160163

161164
### Generate Sample Data

‎distributed-datawarehouse/duckdb/network/config/compute-eu.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Labels:
2+
region: eu

‎distributed-datawarehouse/duckdb/network/config/compute-us.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Labels:
2+
region: us

‎distributed-datawarehouse/duckdb/network/config/compute.yaml

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
NameProvider: hostname
2+
Compute:
3+
Enabled: true
4+
Orchestrators:
5+
- orchestrator:4222
6+
Env:
7+
AllowList:
8+
- AWS_*
9+
UpdateConfig:
10+
Interval: 0
11+
JobAdmissionControl:
12+
AcceptNetworkedJobs: true

‎distributed-datawarehouse/duckdb/network/config/orchestrator.yaml

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
Orchestrator:
2+
Enabled: true
3+
WebUI:
4+
Enabled: true
5+
Labels:
6+
region: global
7+
type: orchestrator
8+
UpdateConfig:
9+
Interval: 0
10+
JobDefaults:
11+
Batch:
12+
Task:
13+
Publisher:
14+
Type: s3
15+
Params:
16+
Bucket: "my-bucket"
17+
Key: jobs/{jobID}/{executionID}
18+
Endpoint: "http://storage:9000"
19+
Region: "storage-region"
20+
Ops:
21+
Task:
22+
Publisher:
23+
Type: s3
24+
Params:
25+
Bucket: "my-bucket"
26+
Key: jobs/{jobID}/{executionID}
27+
Endpoint: "http://storage:9000"
28+
Region: "storage-region"

‎distributed-datawarehouse/duckdb/network/docker-compose.yml

+168
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
name: bacalhau-multi-region
2+
3+
x-common-env-variables: &common-env-variables
4+
MINIO_ROOT_USER: "minioadmin"
5+
MINIO_ROOT_PASSWORD: "minioadmin"
6+
AWS_ACCESS_KEY_ID: "minioadmin"
7+
AWS_SECRET_ACCESS_KEY: "minioadmin"
8+
BACALHAU_DISABLEANALYTICS: true
9+
10+
x-orchestrator-image: &orchestrator-image
11+
image: ghcr.io/bacalhau-project/bacalhau:${ORCHESTRATOR_IMAGE_TAG:-latest}
12+
13+
x-compute-image: &compute-image
14+
image: ghcr.io/bacalhau-project/bacalhau:${COMPUTE_IMAGE_TAG:-latest-dind}
15+
16+
services:
17+
orchestrator:
18+
<<: *orchestrator-image
19+
hostname: orchestrator
20+
command: serve -c /etc/bacalhau/config.yaml --name orchestrator
21+
environment: *common-env-variables
22+
ports:
23+
- "8438:8438"
24+
- "1234:1234"
25+
- "4222:4222"
26+
networks:
27+
- bacalhau-network
28+
volumes:
29+
- ./config/orchestrator.yaml:/etc/bacalhau/config.yaml
30+
healthcheck:
31+
test: ["CMD", "bacalhau", "agent", "alive"]
32+
interval: 2s
33+
timeout: 2s
34+
retries: 10
35+
start_period: 5s
36+
37+
compute-us-1:
38+
<<: *compute-image
39+
hostname: compute-us-1
40+
command: serve -c /etc/bacalhau/config.yaml -c /etc/bacalhau/config-regional.yaml
41+
volumes:
42+
- ./config/compute.yaml:/etc/bacalhau/config.yaml
43+
- ./config/compute-us.yaml:/etc/bacalhau/config-regional.yaml
44+
environment:
45+
<<: *common-env-variables
46+
depends_on:
47+
orchestrator:
48+
condition: service_healthy
49+
storage-us:
50+
condition: service_healthy
51+
privileged: true
52+
networks:
53+
- bacalhau-network
54+
- us-region
55+
56+
compute-us-2:
57+
extends: compute-us-1
58+
hostname: compute-us-2
59+
60+
compute-us-3:
61+
extends: compute-us-1
62+
hostname: compute-us-3
63+
64+
storage-us:
65+
image: quay.io/minio/minio
66+
entrypoint: sh
67+
command: -c 'mkdir -p /data/my-bucket && minio server /data --console-address ":9001"'
68+
environment: *common-env-variables
69+
ports:
70+
- "9002:9000"
71+
- "9003:9001"
72+
networks:
73+
us-region:
74+
aliases:
75+
- storage-local
76+
bacalhau-network:
77+
healthcheck:
78+
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
79+
interval: 2s
80+
timeout: 2s
81+
retries: 10
82+
start_period: 5s
83+
84+
compute-eu-1:
85+
<<: *compute-image
86+
hostname: compute-eu-1
87+
command: serve -c /etc/bacalhau/config.yaml -c /etc/bacalhau/config-regional.yaml
88+
volumes:
89+
- ./config/compute.yaml:/etc/bacalhau/config.yaml
90+
- ./config/compute-eu.yaml:/etc/bacalhau/config-regional.yaml
91+
environment:
92+
<<: *common-env-variables
93+
depends_on:
94+
orchestrator:
95+
condition: service_healthy
96+
storage-eu:
97+
condition: service_healthy
98+
privileged: true
99+
networks:
100+
- bacalhau-network
101+
- eu-region
102+
103+
compute-eu-2:
104+
extends: compute-eu-1
105+
hostname: compute-eu-2
106+
107+
compute-eu-3:
108+
extends: compute-eu-1
109+
hostname: compute-eu-3
110+
111+
storage-eu:
112+
image: quay.io/minio/minio
113+
entrypoint: sh
114+
command: -c 'mkdir -p /data/my-bucket && minio server /data --console-address ":9001"'
115+
environment: *common-env-variables
116+
ports:
117+
- "9004:9000"
118+
- "9005:9001"
119+
networks:
120+
eu-region:
121+
aliases:
122+
- storage-local
123+
bacalhau-network:
124+
healthcheck:
125+
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
126+
interval: 2s
127+
timeout: 2s
128+
retries: 10
129+
start_period: 5s
130+
131+
client:
132+
image: ghcr.io/bacalhau-project/bacalhau:latest
133+
entrypoint: /bin/sh
134+
stdin_open: true
135+
tty: true
136+
stop_signal: SIGTERM
137+
stop_grace_period: 3s
138+
environment:
139+
<<: *common-env-variables
140+
BACALHAU_API_HOST: orchestrator
141+
depends_on:
142+
- orchestrator
143+
networks:
144+
- bacalhau-network
145+
146+
storage-global:
147+
image: quay.io/minio/minio
148+
entrypoint: sh
149+
command: -c 'mkdir -p /data/my-bucket && minio server /data --console-address ":9001"'
150+
environment: *common-env-variables
151+
ports:
152+
- "9000:9000"
153+
- "9001:9001"
154+
networks:
155+
bacalhau-network:
156+
aliases:
157+
- storage
158+
healthcheck:
159+
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
160+
interval: 2s
161+
timeout: 2s
162+
retries: 10
163+
start_period: 5s
164+
165+
networks:
166+
bacalhau-network:
167+
us-region:
168+
eu-region:

0 commit comments

Comments
 (0)