Skip to content

Commit f2b1bb3

Browse files
Multi-language workflows guide small changes
1 parent cea7dcb commit f2b1bb3

File tree

4 files changed

+119
-83
lines changed

4 files changed

+119
-83
lines changed

guides/workflows/multi-language.mdx

Lines changed: 89 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@ icon: diagram-project
1010
</Card>
1111
</CardGroup>
1212

13+
## Tilebox languages and SDKs
14+
15+
Tilebox supports multiple languages and SDKs for running workflows.
16+
All Tilebox SDKs and workflows are designed to be interoperable, which means it's possible to have a workflow where individual tasks are executed in different languages.
17+
Check out [Languages & SDKs](/sdks/introduction) to learn more about currently available programming SDKs.
18+
1319
## Why multi-language workflows?
1420

1521
You might need to use multiple languages in a single workflow for many reasons, such as:
@@ -18,41 +24,42 @@ You might need to use multiple languages in a single workflow for many reasons,
1824
- You want to use a library that is only available in a specific language (for example xarray in Python)
1925
- You started prototyping in Python, but need to start migrating the compute-intensive parts of your workflow to a different language for performance reasons
2026

21-
## Example workflow
27+
## Multi-language workflow example
2228

2329
This guide will tackle the first use case: you have a backend in Go and want to offload some of the processing to Python.
2430

2531
## Defining tasks in Python and Go
2632

2733
```python Python
28-
class TaskingWorkflow(Task):
34+
class ScheduleImageCapture(Task):
2935
# The input parameters must match the ones defined in the Go task
30-
city: str
31-
time: datetime
32-
image_resolution: str
36+
location: tuple[float, float] # lat_lon
37+
resolution_m: int
38+
spectral_bands: list[float] # spectral bands in nm
3339

3440
def execute(self, context: ExecutionContext) -> None:
3541
# Here you can implement your task logic, submit subtasks, etc.
36-
print(f"Tasking workflow executed for {self.city} at {self.time} with resolution {self.image_resolution}")
42+
print(f"Image captured for {self.location} with {self.resolution_m}m resolution and bands {self.spectral_bands}")
3743

3844
@staticmethod
3945
def identifier() -> tuple[str, str]:
4046
# The identifier must match the one defined in the Go task
41-
return "tilebox.com/tasking_workflow", "v1.0"
47+
return "tilebox.com/schedule_image_capture", "v1.0"
4248
```
4349

4450
```go Go
45-
type TaskingWorkflow struct {
46-
City string `json:"city"` // json tags must match the Python task definition
47-
Time time.Time `json:"time"`
48-
ImageResolution string `json:"image_resolution"`
51+
type ScheduleImageCapture struct {
52+
// json tags must match the Python task definition
53+
Location [2]float64 `json:"location"` // lat_lon
54+
ResolutionM int `json:"resolution_m"`
55+
SpectralBands []float64 `json:"spectral_bands"` // spectral bands in nm
4956
}
5057

5158
// No need to define the Execute method since we're only submitting the task
5259

5360
// Identifier must match with the task identifier in the Python runner
54-
func (t *TaskingWorkflow) Identifier() workflows.TaskIdentifier {
55-
return workflows.NewTaskIdentifier("tilebox.com/tasking_workflow", "v1.0")
61+
func (t *ScheduleImageCapture) Identifier() workflows.TaskIdentifier {
62+
return workflows.NewTaskIdentifier("tilebox.com/schedule_image_capture", "v1.0")
5663
}
5764
```
5865

@@ -81,68 +88,89 @@ A couple important points to note:
8188

8289
## Creating a Go server that submits jobs
8390

84-
Write a simple HTTP server in Go with a `/submit` endpoint that accepts requests to submit a `TaskingWorkflow` job.
91+
Write a simple HTTP server in Go with a `/submit` endpoint that accepts requests to submit a `ScheduleImageCapture` job.
8592

8693
<Note>
8794
Both Go and Python code are using `test-cluster-tZD9Ca2qsqt4V` as the cluster slug. You should replace it with your own cluster slug, which you can create in the [Tilebox Console](https://console.tilebox.com/workflows/clusters).
8895
</Note>
8996

9097
```go Go
9198
func main() {
92-
ctx := context.Background()
93-
client := workflows.NewClient()
99+
ctx := context.Background()
100+
client := workflows.NewClient()
94101

95-
cluster, err := client.Clusters.Get(ctx, "test-cluster-tZD9Ca2qsqt4V")
96-
if err != nil {
97-
log.Fatal(err)
98-
}
102+
cluster, err := client.Clusters.Get(ctx, "test-cluster-tZD9Ca2qsqt4V")
103+
if err != nil {
104+
log.Fatal(err)
105+
}
99106

100-
http.HandleFunc("/submit", submitHandler(client, cluster))
107+
http.HandleFunc("/submit", submitHandler(client, cluster))
101108

102-
log.Println("Server starting on http://localhost:8080")
103-
log.Fatal(http.ListenAndServe(":8080", nil))
109+
log.Println("Server starting on http://localhost:8080")
110+
log.Fatal(http.ListenAndServe(":8080", nil))
104111
}
105112

106113
// Submit a job based on some query parameters
107114
func submitHandler(client *workflows.Client, cluster *workflows.Cluster) http.HandlerFunc {
108-
return func(w http.ResponseWriter, r *http.Request) {
109-
city := r.URL.Query().Get("city")
110-
timeArg := r.URL.Query().Get("time")
111-
resolution := r.URL.Query().Get("resolution")
112-
113-
if city == "" {
114-
http.Error(w, "city is required", http.StatusBadRequest)
115-
return
116-
}
117-
118-
taskingTime, err := time.Parse(time.RFC3339, timeArg)
119-
if err != nil {
120-
http.Error(w, err.Error(), http.StatusBadRequest)
121-
return
122-
}
123-
124-
job, err := client.Jobs.Submit(r.Context(), fmt.Sprintf("tasking/%s", city), cluster,
125-
[]workflows.Task{
126-
&TaskingWorkflow{
127-
City: city,
128-
Time: taskingTime,
129-
ImageResolution: resolution,
130-
},
131-
},
132-
)
133-
if err != nil {
134-
http.Error(w, err.Error(), http.StatusInternalServerError)
135-
return
136-
}
137-
138-
_, _ = io.WriteString(w, fmt.Sprintf("Job submitted: %s\n", job.ID))
139-
}
115+
return func(w http.ResponseWriter, r *http.Request) {
116+
latArg := r.URL.Query().Get("lat")
117+
lonArg := r.URL.Query().Get("lon")
118+
resolutionArg := r.URL.Query().Get("resolution")
119+
bandsArg := r.URL.Query().Get("bands[]")
120+
121+
latFloat, err := strconv.ParseFloat(latArg, 64)
122+
if err != nil {
123+
http.Error(w, err.Error(), http.StatusBadRequest)
124+
return
125+
}
126+
lonFloat, err := strconv.ParseFloat(lonArg, 64)
127+
if err != nil {
128+
http.Error(w, err.Error(), http.StatusBadRequest)
129+
return
130+
}
131+
132+
resolutionM, err := strconv.Atoi(resolutionArg)
133+
if err != nil {
134+
http.Error(w, err.Error(), http.StatusBadRequest)
135+
return
136+
}
137+
138+
var spectralBands []float64
139+
for _, bandArg := range strings.Split(bandsArg, ",") {
140+
band, err := strconv.ParseFloat(bandArg, 64)
141+
if err != nil {
142+
http.Error(w, err.Error(), http.StatusBadRequest)
143+
return
144+
}
145+
spectralBands = append(spectralBands, band)
146+
}
147+
148+
job, err := client.Jobs.Submit(r.Context(), "Schedule Image capture", cluster,
149+
[]workflows.Task{
150+
&ScheduleImageCapture{
151+
Location: [2]float64{latFloat, lonFloat},
152+
ResolutionM: resolutionM,
153+
SpectralBands: spectralBands,
154+
},
155+
},
156+
)
157+
if err != nil {
158+
http.Error(w, err.Error(), http.StatusInternalServerError)
159+
return
160+
}
161+
162+
_, _ = io.WriteString(w, fmt.Sprintf("Job submitted: %s\n", job.ID))
163+
}
140164
}
141165
```
142166

167+
<Note>
168+
In the same way that you can submit jobs across languages you can also submit subtasks across languages.
169+
</Note>
170+
143171
## Creating a Python runner
144172

145-
Write a Python script that starts a task runner and registers the `TaskingWorkflow` task.
173+
Write a Python script that starts a task runner and registers the `ScheduleImageCapture` task.
146174

147175
```python Python
148176
from tilebox.workflows import Client
@@ -152,7 +180,7 @@ def main():
152180
runner = client.runner(
153181
"test-cluster-tZD9Ca2qsqt4V",
154182
tasks=[
155-
TaskingWorkflow,
183+
ScheduleImageCapture,
156184
],
157185
)
158186
runner.run_forever()
@@ -161,7 +189,7 @@ if __name__ == "__main__":
161189
main()
162190
```
163191

164-
## Running the workflow
192+
## Testing it
165193

166194
Start the Go server.
167195

@@ -178,13 +206,13 @@ uv run runner.py
178206
Submit a job to the Go server.
179207

180208
```bash Shell
181-
curl http://localhost:8080/submit?city=Zurich&time=2025-05-29T08:06:42Z&resolution=30m
209+
curl http://localhost:8080/submit?lat=40.75&lon=-73.98&resolution=30&bands[]=489.0,560.6,666.5
182210
```
183211

184212
Check the Python runner output, it should print the following line:
185213

186214
```plaintext Output
187-
Tasking workflow executed for Zurich at 2025-05-29T08:06:42Z with resolution 30m
215+
Image captured for [40.75, -73.98] with 30m resolution and bands [489, 560.6, 666.5]
188216
```
189217

190218
## Next Steps
@@ -196,4 +224,4 @@ Tasking workflow executed for Zurich at 2025-05-29T08:06:42Z with resolution 30m
196224
</CardGroup>
197225

198226
As a learning exercise, you can try to change the [News API Workflow](/workflows/concepts/tasks#dependencies-example) to replace the `FetchNews` task with a Go task and keep all the other tasks in Python.
199-
You will learn how to use a Go task in the middle of a workflow implemented in Python.
227+
You'll learn how to submit a subtask in another language than what the current task is executed in.

workflows/caches.mdx

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ The following code shows an example of how cache groups can be used.
206206
from tilebox.workflows import Task, ExecutionContext
207207
import random
208208

209-
class CacheGroupDemoWorkflow(Task):
209+
class CacheGroupDemo(Task):
210210
n: int
211211

212212
def execute(self, context: ExecutionContext):
@@ -254,18 +254,18 @@ class PrintSum(Task):
254254
```
255255
</CodeGroup>
256256

257-
Submitting a job of the `CacheGroupDemoWorkflow` and running it with a task runner can be done as follows:
257+
Submitting a job of the `CacheGroupDemo` and running it with a task runner can be done as follows:
258258

259259
<CodeGroup>
260260
```python Python
261261
# submit a job to test our workflow
262262
job_client = client.jobs()
263-
job_client.submit("cache-groups", CacheGroupDemoWorkflow(5), cluster="dev-cluster")
263+
job_client.submit("cache-groups", CacheGroupDemo(5), cluster="dev-cluster")
264264

265265
# start a runner to execute it
266266
runner = client.runner(
267267
"dev-cluster",
268-
tasks=[CacheGroupDemoWorkflow, ProduceRandomNumbers, ProduceRandomNumber, PrintSum],
268+
tasks=[CacheGroupDemo, ProduceRandomNumbers, ProduceRandomNumber, PrintSum],
269269
cache=LocalFileSystemCache("/path/to/cache/directory"),
270270
)
271271
runner.run_forever()

workflows/concepts/clusters.mdx

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,9 @@ icon: circle-nodes
44
---
55

66
<Accordion title="What is a Cluster?">
7-
Clusters are a logical grouping for [task runners](/workflows/concepts/task-runners). Using clusters, you can scope certain tasks to a specific group of task runners. Tasks, which are always submitted to a specific cluster, are only executed on task runners assigned to the same cluster.
7+
Clusters are a logical grouping for [task runners](/workflows/concepts/task-runners).
8+
Using clusters, you can scope certain tasks to a specific group of task runners.
9+
Tasks, which are always submitted to a specific cluster, are only executed on task runners assigned to the same cluster.
810
</Accordion>
911

1012
## Use Cases
@@ -70,7 +72,8 @@ Cluster(slug='testing-CvufcSxcC9SKfe', display_name='testing')
7072

7173
### Cluster Slug
7274

73-
Each cluster has a unique identifier, combining the cluster's name and an automatically generated identifier. Use this slug to reference the cluster for other operations, like submitting a job or subtasks.
75+
Each cluster has a unique identifier, combining the cluster's name and an automatically generated identifier.
76+
Use this slug to reference the cluster for other operations, like submitting a job or subtasks.
7477

7578
### Listing Clusters
7679

@@ -148,13 +151,16 @@ To delete a cluster, use the `delete` method and pass the cluster's slug:
148151

149152
## Jobs Across Different Clusters
150153

151-
When [submitting a job](/workflows/concepts/jobs), you need to specify which cluster the job's root task should be executed on. This allows you to direct the job to a specific set of task runners. By default, all sub-tasks within a job are also submitted to the same cluster, but this can be overridden to submit sub-tasks to different clusters if needed. See the example below for a job that spans across multiple clusters.
154+
When [submitting a job](/workflows/concepts/jobs), you need to specify which cluster the job's root task should be executed on.
155+
This allows you to direct the job to a specific set of task runners.
156+
By default, all sub-tasks within a job are also submitted to the same cluster, but this can be overridden to submit sub-tasks to different clusters if needed.
157+
See the example below for a job that spans across multiple clusters.
152158

153159
<CodeGroup title="Multi-Cluster Workflow Example">
154160
```python Python
155161
from tilebox.workflows import Task, ExecutionContext, Client
156162

157-
class MultiClusterWorkflow(Task):
163+
class MultiCluster(Task):
158164
def execute(self, context: ExecutionContext) -> None:
159165
# this submits a task to the same cluster as the one currently executing this task
160166
same_cluster = context.submit_subtask(DummyTask())
@@ -176,7 +182,7 @@ client = Client()
176182
job_client = client.jobs()
177183
job = job_client.submit(
178184
"my-job",
179-
MultiClusterWorkflow(),
185+
MultiCluster(),
180186
cluster="testing-CvufcSxcC9SKfe",
181187
)
182188
```
@@ -190,9 +196,9 @@ import (
190196
"github.com/tilebox/tilebox-go/workflows/v1/subtask"
191197
)
192198

193-
type MultiClusterWorkflow struct{}
199+
type MultiCluster struct{}
194200

195-
func (t *MultiClusterWorkflow) Execute(ctx context.Context) error {
201+
func (t *MultiCluster) Execute(ctx context.Context) error {
196202
// this submits a task to the same cluster as the one currently executing this task
197203
sameCluster, err := workflows.SubmitSubtask(ctx, &DummyTask{})
198204
if err != nil {
@@ -227,11 +233,13 @@ func main() {
227233
"my-job",
228234
"testing-CvufcSxcC9SKfe",
229235
[]workflows.Task{
230-
&MultiClusterWorkflow{},
236+
&MultiCluster{},
231237
},
232238
)
233239
}
234240
```
235241
</CodeGroup>
236242

237-
This workflow requires at least two task runners to complete. One must be in the "testing" cluster, and the other must be in the "other-cluster" cluster. If no task runners are available in the "other-cluster," the task submitted to that cluster will remain queued until a task runner is available. It won't execute on a task runner in the "testing" cluster, even if the task runner has the `DummyTask` registered.
243+
This workflow requires at least two task runners to complete. One must be in the "testing" cluster, and the other must be in the "other-cluster" cluster.
244+
If no task runners are available in the "other-cluster," the task submitted to that cluster will remain queued until a task runner is available.
245+
It won't execute on a task runner in the "testing" cluster, even if the task runner has the `DummyTask` registered.

0 commit comments

Comments
 (0)