Skip to content

Commit 3b56c37

Browse files
committed
refactor: Restore parallelism and add backpressure to backfill_parallel
In Obelisk 0.24 the join sets are closed on drop. Original version would thus wait for child execution completion in every iteration of the `for` cycle. This change introduces `join_set_batch` which collects join sets of the same `list-stargazers` batch. Closing it on every `while` iteration adds backpressure.
1 parent 000b315 commit 3b56c37

1 file changed

Lines changed: 9 additions & 8 deletions

File tree

  • workflow/workflow-rs/src

workflow/workflow-rs/src/lib.rs

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -93,21 +93,22 @@ impl Guest for Component {
9393
let mut cursor = None;
9494
while let Some(resp) =
9595
github::account::list_stargazers(&repo, page_size, cursor.as_deref())?
96+
// direct call to an activity
9697
{
98+
let mut join_set_batch = Vec::new();
9799
for login in &resp.logins {
98-
// No need to await the result of the child workflow.
99-
// When this execution completes, all join sets will be awaited.
100-
imported_workflow_ext::star_added_parallel_submit(
101-
&new_join_set_named(login, ClosingStrategy::Complete)
102-
.expect("github login does not contain illegal characters"),
103-
login,
104-
&repo,
105-
);
100+
let join_set = new_join_set_named(login, ClosingStrategy::Complete)
101+
.expect("github login does not contain illegal characters");
102+
// `-submit`-ting child executions without `-await`-ing results
103+
imported_workflow_ext::star_added_parallel_submit(&join_set, login, &repo);
104+
join_set_batch.push(join_set);
106105
}
107106
if resp.logins.len() < usize::from(page_size) {
107+
// last batch gets closed here.
108108
break;
109109
}
110110
cursor = Some(resp.cursor);
111+
// join_set_batch get closed blocking until child executions are completed.
111112
}
112113
Ok(())
113114
}

0 commit comments

Comments
 (0)