Skip to content

Commit fce7165

Browse files
jackpot51jackpot51
committed
Use scoped threads to reduce clones for parallel builds
1 parent 94d184e commit fce7165

File tree

4 files changed

+137
-75
lines changed

4 files changed

+137
-75
lines changed

‎scripts/pop-ci/Cargo.lock

Lines changed: 74 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎scripts/pop-ci/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,6 @@ edition = "2018"
88
[dependencies]
99
async-std = { version = "1.9.0", features = ["unstable"] }
1010
clap = { version = "2.33.3", default-features = false }
11+
crossbeam = "0.8.1"
1112
futures = "0.3.16"
1213
json = "0.12.4"

‎scripts/pop-ci/src/cache.rs

Lines changed: 29 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ use std::{
33
fs,
44
io,
55
path::{Path, PathBuf},
6-
thread,
76
};
87

98
pub struct Cache {
@@ -110,40 +109,44 @@ impl Cache {
110109
}
111110

112111
pub fn build_parallel<
113-
F: Fn(&Path) -> io::Result<()> + Send + 'static
112+
F: Fn(&Path) -> io::Result<()> + Send
114113
>(&mut self, names: BTreeMap<String, F>, force: bool) -> BTreeMap<String, io::Result<(PathBuf, bool)>> {
115-
let mut threads = BTreeMap::new();
116114
let mut results = BTreeMap::new();
117-
for (name, f) in names {
118-
match self.build_inner(&name, force) {
119-
Ok((path, partial_path_opt)) => match partial_path_opt {
120-
Some(partial_path) => {
121-
threads.insert(name, thread::spawn(move || {
122-
f(&partial_path)?;
123-
fs::rename(partial_path, &path)?;
124-
Ok(path)
125-
}));
115+
116+
crossbeam::thread::scope(|s| {
117+
let mut threads = BTreeMap::new();
118+
119+
for (name, f) in names {
120+
match self.build_inner(&name, force) {
121+
Ok((path, partial_path_opt)) => match partial_path_opt {
122+
Some(partial_path) => {
123+
threads.insert(name, s.spawn(move |_| {
124+
f(&partial_path)?;
125+
fs::rename(partial_path, &path)?;
126+
Ok(path)
127+
}));
128+
},
129+
None => {
130+
results.insert(name, Ok((path, false)));
131+
}
126132
},
127-
None => {
128-
results.insert(name, Ok((path, false)));
133+
Err(err) => {
134+
results.insert(name, Err(err));
129135
}
130-
},
131-
Err(err) => {
132-
results.insert(name, Err(err));
133136
}
134137
}
135-
}
136138

137-
for (name, thread) in threads {
138-
match thread.join().unwrap() {
139-
Ok(path) => {
140-
results.insert(name, Ok((path, true)));
141-
},
142-
Err(err) => {
143-
results.insert(name, Err(err));
139+
for (name, thread) in threads {
140+
match thread.join().unwrap() {
141+
Ok(path) => {
142+
results.insert(name, Ok((path, true)));
143+
},
144+
Err(err) => {
145+
results.insert(name, Err(err));
146+
}
144147
}
145148
}
146-
}
149+
}).unwrap();
147150

148151
results
149152
}

‎scripts/pop-ci/src/main.rs

Lines changed: 33 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -439,50 +439,42 @@ sudo sbuild-update \
439439
continue;
440440
}
441441

442-
let github_status = {
443-
//TODO: cleanup extra clones
444-
let commit = commit.clone();
445-
let commit_name = commit_name.clone();
446-
let repo_name = repo_name.clone();
447-
let suite = suite.clone();
448-
let suite_name = suite_name.clone();
449-
move |step: &str, status: &str| {
450-
let target_url = match env::var("BUILD_URL") {
451-
Ok(some) => some,
452-
Err(_) => return,
453-
};
442+
let github_status = |step: &str, status: &str| {
443+
let target_url = match env::var("BUILD_URL") {
444+
Ok(some) => some,
445+
Err(_) => return,
446+
};
454447

455-
eprintln!(
456-
bold!("{}: {}: {}: {} github status {}"),
457-
repo_name, commit_name, suite_name, step, status
458-
);
448+
eprintln!(
449+
bold!("{}: {}: {}: {} github status {}"),
450+
repo_name, commit_name, suite_name, step, status
451+
);
459452

460-
let (context, description) = if dev {
461-
(
462-
format!("ubuntu/staging/{}/{}", suite.id(), step),
463-
format!("Ubuntu Staging {} {}", suite.id(), step),
464-
)
465-
} else {
466-
(
467-
format!("pop-os/staging/{}/{}", suite.id(), step),
468-
format!("Pop!_OS Staging {} {}", suite.id(), step),
469-
)
470-
};
453+
let (context, description) = if dev {
454+
(
455+
format!("ubuntu/staging/{}/{}", suite.id(), step),
456+
format!("Ubuntu Staging {} {}", suite.id(), step),
457+
)
458+
} else {
459+
(
460+
format!("pop-os/staging/{}/{}", suite.id(), step),
461+
format!("Pop!_OS Staging {} {}", suite.id(), step),
462+
)
463+
};
471464

472-
match github_status_inner(
473-
&repo_name,
474-
&commit,
475-
&context,
476-
&description,
477-
status,
478-
&target_url
479-
) {
480-
Ok(()) => (),
481-
Err(err) => eprintln!(
482-
bold!("{}: {}: {}: {} github status {} failed: {}"),
483-
repo_name, commit_name, suite_name, step, status, err
484-
)
485-
}
465+
match github_status_inner(
466+
&repo_name,
467+
&commit,
468+
&context,
469+
&description,
470+
status,
471+
&target_url
472+
) {
473+
Ok(()) => (),
474+
Err(err) => eprintln!(
475+
bold!("{}: {}: {}: {} github status {} failed: {}"),
476+
repo_name, commit_name, suite_name, step, status, err
477+
)
486478
}
487479
};
488480

@@ -724,17 +716,9 @@ sudo sbuild-update \
724716
continue;
725717
}
726718

727-
728-
//TODO: cleanup extra clones
729-
let arch = arch.clone();
730-
let arm64_opt = arm64_opt.map(|x| x.to_string());
731719
let commit_name = commit_name.clone();
732-
let dsc_path = dsc_path.clone();
733-
let github_status = github_status.clone();
734-
let repo_name = repo_name.clone();
735720
let repo_info = repo_info.clone();
736721
let source = source.clone();
737-
let suite = suite.clone();
738722
let suite_name = suite_name.clone();
739723
binary_builds.insert(arch.id().to_string(), move |path: &Path| {
740724
eprintln!(bold!("{}: {}: {}: {}: binary building"), repo_name, commit_name, suite_name, arch.id());

0 commit comments

Comments
 (0)