diff --git a/src/lib.rs b/src/lib.rs index 6d022e30a..f82f72933 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,7 +37,7 @@ mod cache; mod client; mod cmdline; mod commands; -mod compiler; +pub mod compiler; pub mod config; pub mod dist; mod jobserver; diff --git a/src/server.rs b/src/server.rs index 76a63ed27..67a897227 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1540,14 +1540,14 @@ where } } -#[derive(Serialize, Deserialize, Debug, Clone, Default)] +#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)] pub struct PerLanguageCount { counts: HashMap, adv_counts: HashMap, } impl PerLanguageCount { - fn increment(&mut self, kind: &CompilerKind, lang: &Language) { + pub fn increment(&mut self, kind: &CompilerKind, lang: &Language) { let lang_comp_key = kind.lang_comp_kind(lang); let adv_count = self.adv_counts.entry(lang_comp_key).or_insert(0); *adv_count += 1; @@ -1575,7 +1575,7 @@ impl PerLanguageCount { } /// Statistics about the server. -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] pub struct ServerStats { /// The count of client compile requests. pub compile_requests: u64, diff --git a/tests/dist.rs b/tests/dist.rs index cc63f0f18..aa2a0a68e 100644 --- a/tests/dist.rs +++ b/tests/dist.rs @@ -7,8 +7,9 @@ extern crate sccache; extern crate serde_json; use crate::harness::{ - cargo_command, get_stats, init_cargo, sccache_command, start_local_daemon, stop_local_daemon, - write_json_cfg, write_source, + client::{sccache_client_cfg, SccacheClient}, + dist::{cargo_command, sccache_dist_path, DistSystem}, + init_cargo, write_source, }; use assert_cmd::prelude::*; use sccache::config::HTTPUrl; @@ -16,7 +17,6 @@ use sccache::dist::{ AssignJobResult, CompileCommand, InputsReader, JobId, JobState, RunJobResult, ServerIncoming, ServerOutgoing, SubmitToolchainResult, Toolchain, ToolchainReader, }; -use std::ffi::OsStr; use std::path::Path; use std::process::Output; @@ -24,17 +24,12 @@ use sccache::errors::*; mod harness; -fn basic_compile(tmpdir: &Path, sccache_cfg_path: &Path, sccache_cached_cfg_path: &Path) { - let envs: Vec<(_, &OsStr)> = vec![ - ("RUST_BACKTRACE", "1".as_ref()), - ("SCCACHE_LOG", "debug".as_ref()), - ("SCCACHE_CONF", sccache_cfg_path.as_ref()), - ("SCCACHE_CACHED_CONF", sccache_cached_cfg_path.as_ref()), - ]; +fn basic_compile(client: &SccacheClient, tmpdir: &Path) { let source_file = "x.c"; let obj_file = "x.o"; write_source(tmpdir, source_file, "#if !defined(SCCACHE_TEST_DEFINE)\n#error SCCACHE_TEST_DEFINE is not defined\n#endif\nint x() { return 5; }"); - sccache_command() + client + .cmd() .args([ std::env::var("CC") .unwrap_or_else(|_| "gcc".to_string()) @@ -45,21 +40,13 @@ fn basic_compile(tmpdir: &Path, sccache_cfg_path: &Path, sccache_cached_cfg_path .arg(tmpdir.join(source_file)) .arg("-o") .arg(tmpdir.join(obj_file)) - .envs(envs) + .env("RUST_BACKTRACE", "1") + .env("SCCACHE_RECACHE", "1") .assert() .success(); } -fn rust_compile(tmpdir: &Path, sccache_cfg_path: &Path, sccache_cached_cfg_path: &Path) -> Output { - let sccache_path = assert_cmd::cargo::cargo_bin("sccache").into_os_string(); - let envs: Vec<(_, &OsStr)> = vec![ - ("RUSTC_WRAPPER", sccache_path.as_ref()), - ("CARGO_TARGET_DIR", "target".as_ref()), - ("RUST_BACKTRACE", "1".as_ref()), - ("SCCACHE_LOG", "debug".as_ref()), - ("SCCACHE_CONF", sccache_cfg_path.as_ref()), - ("SCCACHE_CACHED_CONF", sccache_cached_cfg_path.as_ref()), - ]; +fn rust_compile(client: &SccacheClient, tmpdir: &Path) -> Output { let cargo_name = "sccache-dist-test"; let cargo_path = init_cargo(tmpdir, cargo_name); @@ -87,7 +74,16 @@ fn rust_compile(tmpdir: &Path, sccache_cfg_path: &Path, sccache_cached_cfg_path: cargo_command() .current_dir(cargo_path) .args(["build", "--release"]) - .envs(envs) + .envs( + client + .cmd() + .get_envs() + .map(|(k, v)| (k, v.unwrap_or_default())), + ) + .env("RUSTC_WRAPPER", &client.path) + .env("CARGO_TARGET_DIR", "target") + .env("RUST_BACKTRACE", "1") + .env("SCCACHE_RECACHE", "1") .output() .unwrap() } @@ -96,7 +92,7 @@ pub fn dist_test_sccache_client_cfg( tmpdir: &Path, scheduler_url: HTTPUrl, ) -> sccache::config::FileConfig { - let mut sccache_cfg = harness::sccache_client_cfg(tmpdir, false); + let mut sccache_cfg = sccache_client_cfg(tmpdir, false); sccache_cfg.cache.disk.as_mut().unwrap().size = 0; sccache_cfg.dist.scheduler_url = Some(scheduler_url); sccache_cfg @@ -110,29 +106,27 @@ fn test_dist_basic() { .tempdir() .unwrap(); let tmpdir = tmpdir.path(); - let sccache_dist = harness::sccache_dist_path(); + let sccache_dist = sccache_dist_path(); - let mut system = harness::DistSystem::new(&sccache_dist, tmpdir); + let mut system = DistSystem::new(&sccache_dist, tmpdir); system.add_scheduler(); system.add_server(); - let sccache_cfg = dist_test_sccache_client_cfg(tmpdir, system.scheduler_url()); - let sccache_cfg_path = tmpdir.join("sccache-cfg.json"); - write_json_cfg(tmpdir, "sccache-cfg.json", &sccache_cfg); - let sccache_cached_cfg_path = tmpdir.join("sccache-cached-cfg"); - - stop_local_daemon(); - start_local_daemon(&sccache_cfg_path, &sccache_cached_cfg_path); - basic_compile(tmpdir, &sccache_cfg_path, &sccache_cached_cfg_path); - - get_stats(|info| { - assert_eq!(1, info.stats.dist_compiles.values().sum::()); - assert_eq!(0, info.stats.dist_errors); - assert_eq!(1, info.stats.compile_requests); - assert_eq!(1, info.stats.requests_executed); - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - }); + let client = system.new_client(&dist_test_sccache_client_cfg( + tmpdir, + system.scheduler_url(), + )); + + basic_compile(&client, tmpdir); + + let stats = client.stats().unwrap(); + + assert_eq!(1, stats.dist_compiles.values().sum::()); + assert_eq!(0, stats.dist_errors); + assert_eq!(1, stats.compile_requests); + assert_eq!(1, stats.requests_executed); + assert_eq!(0, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); } #[test] @@ -143,32 +137,31 @@ fn test_dist_restartedserver() { .tempdir() .unwrap(); let tmpdir = tmpdir.path(); - let sccache_dist = harness::sccache_dist_path(); + let sccache_dist = sccache_dist_path(); - let mut system = harness::DistSystem::new(&sccache_dist, tmpdir); + let mut system = DistSystem::new(&sccache_dist, tmpdir); system.add_scheduler(); let server_handle = system.add_server(); - let sccache_cfg = dist_test_sccache_client_cfg(tmpdir, system.scheduler_url()); - let sccache_cfg_path = tmpdir.join("sccache-cfg.json"); - write_json_cfg(tmpdir, "sccache-cfg.json", &sccache_cfg); - let sccache_cached_cfg_path = tmpdir.join("sccache-cached-cfg"); + let client = system.new_client(&dist_test_sccache_client_cfg( + tmpdir, + system.scheduler_url(), + )); - stop_local_daemon(); - start_local_daemon(&sccache_cfg_path, &sccache_cached_cfg_path); - basic_compile(tmpdir, &sccache_cfg_path, &sccache_cached_cfg_path); + basic_compile(&client, tmpdir); system.restart_server(&server_handle); - basic_compile(tmpdir, &sccache_cfg_path, &sccache_cached_cfg_path); - - get_stats(|info| { - assert_eq!(2, info.stats.dist_compiles.values().sum::()); - assert_eq!(0, info.stats.dist_errors); - assert_eq!(2, info.stats.compile_requests); - assert_eq!(2, info.stats.requests_executed); - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(2, info.stats.cache_misses.all()); - }); + + basic_compile(&client, tmpdir); + + let stats = client.stats().unwrap(); + + assert_eq!(2, stats.dist_compiles.values().sum::()); + assert_eq!(0, stats.dist_errors); + assert_eq!(2, stats.compile_requests); + assert_eq!(2, stats.requests_executed); + assert_eq!(0, stats.cache_hits.all()); + assert_eq!(2, stats.cache_misses.all()); } #[test] @@ -179,28 +172,26 @@ fn test_dist_nobuilder() { .tempdir() .unwrap(); let tmpdir = tmpdir.path(); - let sccache_dist = harness::sccache_dist_path(); + let sccache_dist = sccache_dist_path(); - let mut system = harness::DistSystem::new(&sccache_dist, tmpdir); + let mut system = DistSystem::new(&sccache_dist, tmpdir); system.add_scheduler(); - let sccache_cfg = dist_test_sccache_client_cfg(tmpdir, system.scheduler_url()); - let sccache_cfg_path = tmpdir.join("sccache-cfg.json"); - write_json_cfg(tmpdir, "sccache-cfg.json", &sccache_cfg); - let sccache_cached_cfg_path = tmpdir.join("sccache-cached-cfg"); - - stop_local_daemon(); - start_local_daemon(&sccache_cfg_path, &sccache_cached_cfg_path); - basic_compile(tmpdir, &sccache_cfg_path, &sccache_cached_cfg_path); - - get_stats(|info| { - assert_eq!(0, info.stats.dist_compiles.values().sum::()); - assert_eq!(1, info.stats.dist_errors); - assert_eq!(1, info.stats.compile_requests); - assert_eq!(1, info.stats.requests_executed); - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - }); + let client = system.new_client(&dist_test_sccache_client_cfg( + tmpdir, + system.scheduler_url(), + )); + + basic_compile(&client, tmpdir); + + let stats = client.stats().unwrap(); + + assert_eq!(0, stats.dist_compiles.values().sum::()); + assert_eq!(1, stats.dist_errors); + assert_eq!(1, stats.compile_requests); + assert_eq!(1, stats.requests_executed); + assert_eq!(0, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); } struct FailingServer; @@ -244,97 +235,64 @@ fn test_dist_failingserver() { .tempdir() .unwrap(); let tmpdir = tmpdir.path(); - let sccache_dist = harness::sccache_dist_path(); + let sccache_dist = sccache_dist_path(); - let mut system = harness::DistSystem::new(&sccache_dist, tmpdir); + let mut system = DistSystem::new(&sccache_dist, tmpdir); system.add_scheduler(); system.add_custom_server(FailingServer); - let sccache_cfg = dist_test_sccache_client_cfg(tmpdir, system.scheduler_url()); - let sccache_cfg_path = tmpdir.join("sccache-cfg.json"); - write_json_cfg(tmpdir, "sccache-cfg.json", &sccache_cfg); - let sccache_cached_cfg_path = tmpdir.join("sccache-cached-cfg"); - - stop_local_daemon(); - start_local_daemon(&sccache_cfg_path, &sccache_cached_cfg_path); - basic_compile(tmpdir, &sccache_cfg_path, &sccache_cached_cfg_path); - - get_stats(|info| { - assert_eq!(0, info.stats.dist_compiles.values().sum::()); - assert_eq!(1, info.stats.dist_errors); - assert_eq!(1, info.stats.compile_requests); - assert_eq!(1, info.stats.requests_executed); - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - }); -} - -#[test] -#[cfg_attr(not(feature = "dist-tests"), ignore)] -fn test_dist_cargo_build() { - let tmpdir = tempfile::Builder::new() - .prefix("sccache_dist_test") - .tempdir() - .unwrap(); - let tmpdir = tmpdir.path(); - let sccache_dist = harness::sccache_dist_path(); + let client = system.new_client(&dist_test_sccache_client_cfg( + tmpdir, + system.scheduler_url(), + )); - let mut system = harness::DistSystem::new(&sccache_dist, tmpdir); - system.add_scheduler(); - let _server_handle = system.add_server(); + basic_compile(&client, tmpdir); - let sccache_cfg = dist_test_sccache_client_cfg(tmpdir, system.scheduler_url()); - let sccache_cfg_path = tmpdir.join("sccache-cfg.json"); - write_json_cfg(tmpdir, "sccache-cfg.json", &sccache_cfg); - let sccache_cached_cfg_path = tmpdir.join("sccache-cached-cfg"); + let stats = client.stats().unwrap(); - stop_local_daemon(); - start_local_daemon(&sccache_cfg_path, &sccache_cached_cfg_path); - rust_compile(tmpdir, &sccache_cfg_path, &sccache_cached_cfg_path) - .assert() - .success(); - get_stats(|info| { - assert_eq!(1, info.stats.dist_compiles.values().sum::()); - assert_eq!(0, info.stats.dist_errors); - assert_eq!(5, info.stats.compile_requests); - assert_eq!(1, info.stats.requests_executed); - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - }); + assert_eq!(0, stats.dist_compiles.values().sum::()); + assert_eq!(1, stats.dist_errors); + assert_eq!(1, stats.compile_requests); + assert_eq!(1, stats.requests_executed); + assert_eq!(0, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); } #[test] #[cfg_attr(not(feature = "dist-tests"), ignore)] -fn test_dist_cargo_makeflags() { +fn test_dist_cargo_build() { let tmpdir = tempfile::Builder::new() .prefix("sccache_dist_test") .tempdir() .unwrap(); let tmpdir = tmpdir.path(); - let sccache_dist = harness::sccache_dist_path(); + let sccache_dist = sccache_dist_path(); - let mut system = harness::DistSystem::new(&sccache_dist, tmpdir); + let mut system = DistSystem::new(&sccache_dist, tmpdir); system.add_scheduler(); let _server_handle = system.add_server(); - let sccache_cfg = dist_test_sccache_client_cfg(tmpdir, system.scheduler_url()); - let sccache_cfg_path = tmpdir.join("sccache-cfg.json"); - write_json_cfg(tmpdir, "sccache-cfg.json", &sccache_cfg); - let sccache_cached_cfg_path = tmpdir.join("sccache-cached-cfg"); + let client = system.new_client(&dist_test_sccache_client_cfg( + tmpdir, + system.scheduler_url(), + )); - stop_local_daemon(); - start_local_daemon(&sccache_cfg_path, &sccache_cached_cfg_path); - let compile_output = rust_compile(tmpdir, &sccache_cfg_path, &sccache_cached_cfg_path); + let compile_output = rust_compile(&client, tmpdir); + // Ensure sccache ignores inherited jobservers in CARGO_MAKEFLAGS assert!(!String::from_utf8_lossy(&compile_output.stderr) .contains("warning: failed to connect to jobserver from environment variable")); - get_stats(|info| { - assert_eq!(1, info.stats.dist_compiles.values().sum::()); - assert_eq!(0, info.stats.dist_errors); - assert_eq!(5, info.stats.compile_requests); - assert_eq!(1, info.stats.requests_executed); - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - }); + // Assert compilation succeeded + compile_output.assert().success(); + + let stats = client.stats().unwrap(); + + assert_eq!(1, stats.dist_compiles.values().sum::()); + assert_eq!(0, stats.dist_errors); + // check >= 5 because cargo >=1.82 does additional requests with -vV + assert!(stats.compile_requests >= 5); + assert_eq!(1, stats.requests_executed); + assert_eq!(0, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); } diff --git a/tests/harness/client.rs b/tests/harness/client.rs new file mode 100644 index 000000000..c73f81571 --- /dev/null +++ b/tests/harness/client.rs @@ -0,0 +1,177 @@ +use fs_err as fs; +use std::{ + env, + ffi::OsString, + path::{Path, PathBuf}, + process::{Command, Stdio}, + str, + sync::atomic::{AtomicU16, Ordering}, +}; + +use sccache::{ + config::{CacheConfigs, DiskCacheConfig, DistConfig, FileConfig, PreprocessorCacheModeConfig}, + server::{ServerInfo, ServerStats}, +}; + +use super::{prune_command, TC_CACHE_SIZE}; + +pub fn sccache_client_cfg(tmpdir: &Path, preprocessor_cache_mode: bool) -> FileConfig { + let cache_relpath = "client-cache"; + let dist_cache_relpath = "client-dist-cache"; + fs::create_dir_all(tmpdir.join(cache_relpath)).unwrap(); + fs::create_dir_all(tmpdir.join(dist_cache_relpath)).unwrap(); + + let disk_cache = DiskCacheConfig { + dir: tmpdir.join(cache_relpath), + preprocessor_cache_mode: PreprocessorCacheModeConfig { + use_preprocessor_cache_mode: preprocessor_cache_mode, + ..Default::default() + }, + ..Default::default() + }; + FileConfig { + cache: CacheConfigs { + azure: None, + disk: Some(disk_cache), + gcs: None, + gha: None, + memcached: None, + redis: None, + s3: None, + webdav: None, + oss: None, + }, + dist: DistConfig { + auth: Default::default(), // dangerously_insecure + scheduler_url: None, + cache_dir: tmpdir.join(dist_cache_relpath), + toolchains: vec![], + toolchain_cache_size: TC_CACHE_SIZE, + rewrite_includes_only: false, // TODO + }, + server_startup_timeout_ms: None, + } +} + +static CLIENT_PORT: AtomicU16 = AtomicU16::new(4227); + +pub struct SccacheClient { + envvars: Vec<(OsString, OsString)>, + pub path: PathBuf, +} + +#[allow(unused)] +impl SccacheClient { + pub fn new_no_cfg() -> Self { + let path = assert_cmd::cargo::cargo_bin("sccache"); + let port = CLIENT_PORT.fetch_add(1, Ordering::SeqCst); + + let mut envvars = vec![ + ("SCCACHE_SERVER_PORT".into(), port.to_string().into()), + ("TOKIO_WORKER_THREADS".into(), "2".into()), + ]; + + // Send daemon logs to a file if SCCACHE_DEBUG is defined + if env::var("SCCACHE_DEBUG").is_ok() { + envvars.extend_from_slice(&[ + // Allow overriding log level + ( + "SCCACHE_SERVER_LOG".into(), + env::var_os("SCCACHE_SERVER_LOG") + .or(env::var_os("SCCACHE_LOG")) + .unwrap_or("sccache=trace".into()), + ), + // Allow overriding log output path + ( + "SCCACHE_ERROR_LOG".into(), + env::var_os("SCCACHE_ERROR_LOG").unwrap_or( + env::temp_dir() + .join(format!("sccache_local_daemon.{port}.txt")) + .into_os_string(), + ), + ), + ]); + } + + Self { envvars, path } + } + + pub fn new(cfg_path: &Path, cached_cfg_path: &Path) -> Self { + let mut this = Self::new_no_cfg(); + this.envvars.push(("SCCACHE_CONF".into(), cfg_path.into())); + this.envvars + .push(("SCCACHE_CACHED_CONF".into(), cached_cfg_path.into())); + this + } + + pub fn start(self) -> Self { + trace!("sccache --start-server"); + // Don't run this with run() because on Windows `wait_with_output` + // will hang because the internal server process is not detached. + if !self + .cmd() + .arg("--start-server") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .unwrap() + .success() + { + panic!("Failed to start local daemon"); + } + self + } + + pub fn stop(&self) -> bool { + trace!("sccache --stop-server"); + self.cmd() + .arg("--stop-server") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()) + } + + pub fn cmd(&self) -> Command { + let mut cmd = prune_command(Command::new(assert_cmd::cargo::cargo_bin("sccache"))); + cmd.envs( + self.envvars + .iter() + .map(|(k, v)| (k.as_os_str(), v.as_os_str())), + ); + cmd + } + + pub fn info(&self) -> sccache::errors::Result { + self.cmd() + .args(["--show-stats", "--stats-format=json"]) + .output() + .map_err(anyhow::Error::new) + .map_err(|e| e.context("`sccache --show-stats --stats-format=json` failed")) + .map(|output| { + let s = str::from_utf8(&output.stdout).expect("Output not UTF-8"); + serde_json::from_str(s).expect("Failed to parse JSON stats") + }) + } + + pub fn stats(&self) -> sccache::errors::Result { + self.info().map(|info| info.stats) + } + + pub fn zero_stats(&self) { + trace!("sccache --zero-stats"); + drop( + self.cmd() + .arg("--zero-stats") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(), + ); + } +} + +impl Drop for SccacheClient { + fn drop(&mut self) { + self.stop(); + } +} diff --git a/tests/harness/dist.rs b/tests/harness/dist.rs new file mode 100644 index 000000000..eb95c1a8d --- /dev/null +++ b/tests/harness/dist.rs @@ -0,0 +1,557 @@ +use fs_err as fs; +use sccache::config::FileConfig; +#[cfg(any(feature = "dist-client", feature = "dist-server"))] +use sccache::config::HTTPUrl; +use sccache::dist::{self, SchedulerStatusResult, ServerId}; +use std::env; +use std::io::Write; +use std::net::{self, IpAddr, SocketAddr}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::str::{self, FromStr}; +use std::thread; +use std::time::{Duration, Instant}; + +use nix::{ + sys::{ + signal::Signal, + wait::{WaitPidFlag, WaitStatus}, + }, + unistd::{ForkResult, Pid}, +}; +use uuid::Uuid; + +use super::client::SccacheClient; +use super::{prune_command, write_json_cfg, TC_CACHE_SIZE}; + +const CONTAINER_NAME_PREFIX: &str = "sccache_dist_test"; +const DIST_IMAGE: &str = "sccache_dist_test_image"; +const DIST_DOCKERFILE: &str = include_str!("Dockerfile.sccache-dist"); +const DIST_IMAGE_BWRAP_PATH: &str = "/usr/bin/bwrap"; +const MAX_STARTUP_WAIT: Duration = Duration::from_secs(5); + +const DIST_SERVER_TOKEN: &str = "THIS IS THE TEST TOKEN"; + +const CONFIGS_CONTAINER_PATH: &str = "/sccache-bits"; +const BUILD_DIR_CONTAINER_PATH: &str = "/sccache-bits/build-dir"; +const SCHEDULER_PORT: u16 = 10500; +const SERVER_PORT: u16 = 12345; // arbitrary + +pub fn cargo_command() -> Command { + prune_command(Command::new("cargo")) +} + +pub fn sccache_dist_path() -> PathBuf { + assert_cmd::cargo::cargo_bin("sccache-dist") +} + +fn sccache_scheduler_cfg() -> sccache::config::scheduler::Config { + sccache::config::scheduler::Config { + public_addr: SocketAddr::from(([0, 0, 0, 0], SCHEDULER_PORT)), + client_auth: sccache::config::scheduler::ClientAuth::Insecure, + server_auth: sccache::config::scheduler::ServerAuth::Token { + token: DIST_SERVER_TOKEN.to_owned(), + }, + } +} + +fn sccache_server_cfg( + tmpdir: &Path, + scheduler_url: HTTPUrl, + server_ip: IpAddr, +) -> sccache::config::server::Config { + let relpath = "server-cache"; + fs::create_dir(tmpdir.join(relpath)).unwrap(); + + sccache::config::server::Config { + builder: sccache::config::server::BuilderType::Overlay { + build_dir: BUILD_DIR_CONTAINER_PATH.into(), + bwrap_path: DIST_IMAGE_BWRAP_PATH.into(), + }, + cache_dir: Path::new(CONFIGS_CONTAINER_PATH).join(relpath), + public_addr: SocketAddr::new(server_ip, SERVER_PORT), + bind_address: Some(SocketAddr::from(([0, 0, 0, 0], SERVER_PORT))), + scheduler_url, + scheduler_auth: sccache::config::server::SchedulerAuth::Token { + token: DIST_SERVER_TOKEN.to_owned(), + }, + toolchain_cache_size: TC_CACHE_SIZE, + } +} + +// TODO: this is copied from the sccache-dist binary - it's not clear where would be a better place to put the +// code so that it can be included here +fn create_server_token(server_id: ServerId, auth_token: &str) -> String { + format!("{} {}", server_id.addr(), auth_token) +} + +pub enum ServerHandle { + Container { + cid: String, + url: HTTPUrl, + }, + #[allow(dead_code)] + Process { + pid: Pid, + url: HTTPUrl, + }, +} + +pub struct DistSystem { + sccache_dist: PathBuf, + tmpdir: PathBuf, + + scheduler_name: Option, + server_names: Vec, + server_pids: Vec, +} + +impl DistSystem { + pub fn new(sccache_dist: &Path, tmpdir: &Path) -> Self { + // Make sure the docker image is available, building it if necessary + let mut child = Command::new("docker") + .args(["build", "-q", "-t", DIST_IMAGE, "-"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .as_mut() + .unwrap() + .write_all(DIST_DOCKERFILE.as_bytes()) + .unwrap(); + let output = child.wait_with_output().unwrap(); + check_output(&output); + + let tmpdir = tmpdir.join("distsystem"); + fs::create_dir(&tmpdir).unwrap(); + + Self { + sccache_dist: sccache_dist.to_owned(), + tmpdir, + + scheduler_name: None, + server_names: vec![], + server_pids: vec![], + } + } + + pub fn add_scheduler(&mut self) { + let scheduler_cfg_relpath = "scheduler-cfg.json"; + let scheduler_cfg_path = self.tmpdir.join(scheduler_cfg_relpath); + let scheduler_cfg_container_path = + Path::new(CONFIGS_CONTAINER_PATH).join(scheduler_cfg_relpath); + let scheduler_cfg = sccache_scheduler_cfg(); + fs::File::create(scheduler_cfg_path) + .unwrap() + .write_all(&serde_json::to_vec(&scheduler_cfg).unwrap()) + .unwrap(); + + // Create the scheduler + let scheduler_name = make_container_name("scheduler"); + let output = Command::new("docker") + .args([ + "run", + "--name", + &scheduler_name, + "-e", + "SCCACHE_NO_DAEMON=1", + "-e", + "SCCACHE_LOG=debug", + "-e", + "RUST_BACKTRACE=1", + "--network", + "host", + "-v", + &format!("{}:/sccache-dist:z", self.sccache_dist.to_str().unwrap()), + "-v", + &format!( + "{}:{}:z", + self.tmpdir.to_str().unwrap(), + CONFIGS_CONTAINER_PATH + ), + "-d", + DIST_IMAGE, + "bash", + "-c", + &format!( + r#" + set -o errexit && + exec /sccache-dist scheduler --config {cfg} + "#, + cfg = scheduler_cfg_container_path.to_str().unwrap() + ), + ]) + .output() + .unwrap(); + self.scheduler_name = Some(scheduler_name); + + check_output(&output); + + let scheduler_url = self.scheduler_url(); + wait_for_http(scheduler_url, Duration::from_millis(100), MAX_STARTUP_WAIT); + wait_for( + || { + let status = self.scheduler_status(); + if matches!( + status, + SchedulerStatusResult { + num_servers: 0, + num_cpus: _, + in_progress: 0 + } + ) { + Ok(()) + } else { + Err(format!("{:?}", status)) + } + }, + Duration::from_millis(100), + MAX_STARTUP_WAIT, + ); + } + + pub fn add_server(&mut self) -> ServerHandle { + let server_cfg_relpath = format!("server-cfg-{}.json", self.server_names.len()); + let server_cfg_path = self.tmpdir.join(&server_cfg_relpath); + let server_cfg_container_path = Path::new(CONFIGS_CONTAINER_PATH).join(server_cfg_relpath); + + let server_name = make_container_name("server"); + let output = Command::new("docker") + .args([ + "run", + // Important for the bubblewrap builder + "--privileged", + "--name", + &server_name, + "-e", + "SCCACHE_LOG=debug", + "-e", + "RUST_BACKTRACE=1", + "--network", + "host", + "-v", + &format!("{}:/sccache-dist:z", self.sccache_dist.to_str().unwrap()), + "-v", + &format!( + "{}:{}:z", + self.tmpdir.to_str().unwrap(), + CONFIGS_CONTAINER_PATH + ), + "-d", + DIST_IMAGE, + "bash", + "-c", + &format!( + r#" + set -o errexit && + while [ ! -f {cfg}.ready ]; do sleep 0.1; done && + exec /sccache-dist server --config {cfg} + "#, + cfg = server_cfg_container_path.to_str().unwrap() + ), + ]) + .output() + .unwrap(); + self.server_names.push(server_name.clone()); + + check_output(&output); + + let server_ip = IpAddr::from_str("127.0.0.1").unwrap(); + let server_cfg = sccache_server_cfg(&self.tmpdir, self.scheduler_url(), server_ip); + fs::File::create(&server_cfg_path) + .unwrap() + .write_all(&serde_json::to_vec(&server_cfg).unwrap()) + .unwrap(); + fs::File::create(format!("{}.ready", server_cfg_path.to_str().unwrap())).unwrap(); + + let url = HTTPUrl::from_url( + reqwest::Url::parse(&format!("https://{}:{}", server_ip, SERVER_PORT)).unwrap(), + ); + let handle = ServerHandle::Container { + cid: server_name, + url, + }; + self.wait_server_ready(&handle); + handle + } + + pub fn add_custom_server( + &mut self, + handler: S, + ) -> ServerHandle { + let server_addr = { + let ip = IpAddr::from_str("127.0.0.1").unwrap(); + let listener = net::TcpListener::bind(SocketAddr::from((ip, 0))).unwrap(); + listener.local_addr().unwrap() + }; + let token = create_server_token(ServerId::new(server_addr), DIST_SERVER_TOKEN); + let server = dist::http::Server::new( + server_addr, + Some(SocketAddr::from(([0, 0, 0, 0], server_addr.port()))), + self.scheduler_url().to_url(), + token, + handler, + ) + .unwrap(); + let pid = match unsafe { nix::unistd::fork() }.unwrap() { + ForkResult::Parent { child } => { + self.server_pids.push(child); + child + } + ForkResult::Child => { + env::set_var("SCCACHE_LOG", "sccache=trace"); + env_logger::try_init().unwrap(); + server.start().unwrap(); + unreachable!(); + } + }; + + let url = + HTTPUrl::from_url(reqwest::Url::parse(&format!("https://{}", server_addr)).unwrap()); + let handle = ServerHandle::Process { pid, url }; + self.wait_server_ready(&handle); + handle + } + + pub fn restart_server(&mut self, handle: &ServerHandle) { + match handle { + ServerHandle::Container { cid, url: _ } => { + let output = Command::new("docker") + .args(["restart", cid]) + .output() + .unwrap(); + check_output(&output); + } + ServerHandle::Process { pid: _, url: _ } => { + // TODO: pretty easy, just no need yet + panic!("restart not yet implemented for pids") + } + } + self.wait_server_ready(handle) + } + + pub fn wait_server_ready(&mut self, handle: &ServerHandle) { + let url = match handle { + ServerHandle::Container { cid: _, url } | ServerHandle::Process { pid: _, url } => { + url.clone() + } + }; + wait_for_http(url, Duration::from_millis(100), MAX_STARTUP_WAIT); + wait_for( + || { + let status = self.scheduler_status(); + if matches!( + status, + SchedulerStatusResult { + num_servers: 1, + num_cpus: _, + in_progress: 0 + } + ) { + Ok(()) + } else { + Err(format!("{:?}", status)) + } + }, + Duration::from_millis(100), + MAX_STARTUP_WAIT, + ); + } + + pub fn scheduler_url(&self) -> HTTPUrl { + let url = format!("http://127.0.0.1:{}", SCHEDULER_PORT); + HTTPUrl::from_url(reqwest::Url::parse(&url).unwrap()) + } + + fn scheduler_status(&self) -> SchedulerStatusResult { + let res = reqwest::blocking::get(dist::http::urls::scheduler_status( + &self.scheduler_url().to_url(), + )) + .unwrap(); + assert!(res.status().is_success()); + bincode::deserialize_from(res).unwrap() + } + + pub fn new_client(&self, client_config: &FileConfig) -> SccacheClient { + write_json_cfg(&self.tmpdir, "sccache-client.json", client_config); + SccacheClient::new( + &self.tmpdir.join("sccache-client.json"), + &self.tmpdir.join("sccache-cached-cfg"), + ) + .start() + } +} + +// If you want containers to hang around (e.g. for debugging), comment out the "rm -f" lines +impl Drop for DistSystem { + fn drop(&mut self) { + let mut did_err = false; + + // Panicking halfway through drop would either abort (if it's a double panic) or leave us with + // resources that aren't yet cleaned up. Instead, do as much as possible then decide what to do + // at the end - panic (if not already doing so) or let the panic continue + macro_rules! droperr { + ($e:expr) => { + match $e { + Ok(()) => (), + Err(e) => { + did_err = true; + eprintln!("Error with {}: {}", stringify!($e), e) + } + } + }; + } + + let mut logs = vec![]; + let mut outputs = vec![]; + let mut exits = vec![]; + + if let Some(scheduler_name) = self.scheduler_name.as_ref() { + droperr!(Command::new("docker") + .args(["logs", scheduler_name]) + .output() + .map(|o| logs.push((scheduler_name, o)))); + droperr!(Command::new("docker") + .args(["kill", scheduler_name]) + .output() + .map(|o| outputs.push((scheduler_name, o)))); + droperr!(Command::new("docker") + .args(["rm", "-f", scheduler_name]) + .output() + .map(|o| outputs.push((scheduler_name, o)))); + } + for server_name in self.server_names.iter() { + droperr!(Command::new("docker") + .args(["logs", server_name]) + .output() + .map(|o| logs.push((server_name, o)))); + droperr!(Command::new("docker") + .args(["kill", server_name]) + .output() + .map(|o| outputs.push((server_name, o)))); + droperr!(Command::new("docker") + .args(["rm", "-f", server_name]) + .output() + .map(|o| outputs.push((server_name, o)))); + } + for &pid in self.server_pids.iter() { + droperr!(nix::sys::signal::kill(pid, Signal::SIGINT)); + thread::sleep(Duration::from_millis(100)); + let mut killagain = true; // Default to trying to kill again, e.g. if there was an error waiting on the pid + droperr!( + nix::sys::wait::waitpid(pid, Some(WaitPidFlag::WNOHANG)).map(|ws| { + if ws != WaitStatus::StillAlive { + killagain = false; + exits.push(ws) + } + }) + ); + if killagain { + eprintln!("SIGINT didn't kill process, trying SIGKILL"); + droperr!(nix::sys::signal::kill(pid, Signal::SIGKILL)); + droperr!(nix::sys::wait::waitpid(pid, Some(WaitPidFlag::WNOHANG)) + .map_err(|e| e.to_string()) + .and_then(|ws| if ws == WaitStatus::StillAlive { + Err("process alive after sigkill".to_owned()) + } else { + exits.push(ws); + Ok(()) + })); + } + } + + for ( + container, + Output { + status, + stdout, + stderr, + }, + ) in logs + { + println!( + "LOGS == ({}) ==\n> {} <:\n## STDOUT\n{}\n\n## STDERR\n{}\n====", + status, + container, + String::from_utf8_lossy(&stdout), + String::from_utf8_lossy(&stderr) + ); + } + for ( + container, + Output { + status, + stdout, + stderr, + }, + ) in outputs + { + println!( + "OUTPUTS == ({}) ==\n> {} <:\n## STDOUT\n{}\n\n## STDERR\n{}\n====", + status, + container, + String::from_utf8_lossy(&stdout), + String::from_utf8_lossy(&stderr) + ); + } + for exit in exits { + println!("EXIT: {:?}", exit) + } + + if did_err && !thread::panicking() { + panic!("Encountered failures during dist system teardown") + } + } +} + +fn make_container_name(tag: &str) -> String { + format!( + "{}_{}_{}", + CONTAINER_NAME_PREFIX, + tag, + Uuid::new_v4().hyphenated() + ) +} + +fn check_output(output: &Output) { + if !output.status.success() { + println!("{}\n\n[BEGIN STDOUT]\n===========\n{}\n===========\n[FIN STDOUT]\n\n[BEGIN STDERR]\n===========\n{}\n===========\n[FIN STDERR]\n\n", + output.status, String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); + panic!() + } +} + +fn wait_for_http(url: HTTPUrl, interval: Duration, max_wait: Duration) { + // TODO: after upgrading to reqwest >= 0.9, use 'danger_accept_invalid_certs' and stick with that rather than tcp + wait_for( + || { + let url = url.to_url(); + let url = url.socket_addrs(|| None).unwrap(); + match net::TcpStream::connect(url.as_slice()) { + Ok(_) => Ok(()), + Err(e) => Err(e.to_string()), + } + }, + interval, + max_wait, + ) +} + +fn wait_for Result<(), String>>(f: F, interval: Duration, max_wait: Duration) { + let start = Instant::now(); + let mut lasterr; + loop { + match f() { + Ok(()) => return, + Err(e) => lasterr = e, + } + if start.elapsed() > max_wait { + break; + } + thread::sleep(interval) + } + panic!("wait timed out, last error result: {}", lasterr) +} diff --git a/tests/harness/mod.rs b/tests/harness/mod.rs index d148793f2..28452b18c 100644 --- a/tests/harness/mod.rs +++ b/tests/harness/mod.rs @@ -1,114 +1,18 @@ use fs_err as fs; -#[cfg(any(feature = "dist-client", feature = "dist-server"))] -use sccache::config::HTTPUrl; -use sccache::dist::{self, SchedulerStatusResult, ServerId}; -use sccache::server::ServerInfo; use std::env; use std::io::Write; -use std::net::{self, IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; -use std::process::{Command, Output, Stdio}; -use std::str::{self, FromStr}; -use std::thread; -use std::time::{Duration, Instant}; +use std::process::Command; +use std::str; -use assert_cmd::prelude::*; -#[cfg(feature = "dist-server")] -use nix::{ - sys::{ - signal::Signal, - wait::{WaitPidFlag, WaitStatus}, - }, - unistd::{ForkResult, Pid}, -}; -use predicates::prelude::*; use serde::Serialize; -use uuid::Uuid; - -const CONTAINER_NAME_PREFIX: &str = "sccache_dist_test"; -const DIST_IMAGE: &str = "sccache_dist_test_image"; -const DIST_DOCKERFILE: &str = include_str!("Dockerfile.sccache-dist"); -const DIST_IMAGE_BWRAP_PATH: &str = "/usr/bin/bwrap"; -const MAX_STARTUP_WAIT: Duration = Duration::from_secs(5); - -const DIST_SERVER_TOKEN: &str = "THIS IS THE TEST TOKEN"; - -const CONFIGS_CONTAINER_PATH: &str = "/sccache-bits"; -const BUILD_DIR_CONTAINER_PATH: &str = "/sccache-bits/build-dir"; -const SCHEDULER_PORT: u16 = 10500; -const SERVER_PORT: u16 = 12345; // arbitrary -const TC_CACHE_SIZE: u64 = 1024 * 1024 * 1024; // 1 gig +pub mod client; -pub fn start_local_daemon(cfg_path: &Path, cached_cfg_path: &Path) { - // Don't run this with run() because on Windows `wait_with_output` - // will hang because the internal server process is not detached. - if !sccache_command() - .arg("--start-server") - // Uncomment following lines to debug locally. - // .env("SCCACHE_LOG", "sccache=trace") - // .env("RUST_LOG_STYLE", "never") - // .env( - // "SCCACHE_ERROR_LOG", - // env::temp_dir().join("sccache_local_daemon.txt"), - // ) - .env("SCCACHE_CONF", cfg_path) - .env("SCCACHE_CACHED_CONF", cached_cfg_path) - .status() - .unwrap() - .success() - { - panic!("Failed to start local daemon"); - } -} - -pub fn stop_local_daemon() -> bool { - trace!("sccache --stop-server"); - sccache_command() - .arg("--stop-server") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .is_ok_and(|status| status.success()) -} - -pub fn get_stats(f: F) { - sccache_command() - .args(["--show-stats", "--stats-format=json"]) - .assert() - .success() - .stdout(predicate::function(move |output: &[u8]| { - let s = str::from_utf8(output).expect("Output not UTF-8"); - let stats = serde_json::from_str(s).expect("Failed to parse JSON stats"); - eprintln!("get server stats: {stats:?}"); - f(stats); - true - })); -} - -#[allow(unused)] -pub fn zero_stats() { - trace!("sccache --zero-stats"); - drop( - sccache_command() - .arg("--zero-stats") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(), - ); -} - -pub fn write_json_cfg(path: &Path, filename: &str, contents: &T) { - let p = path.join(filename); - let mut f = fs::File::create(p).unwrap(); - f.write_all(&serde_json::to_vec(contents).unwrap()).unwrap(); -} +#[cfg(feature = "dist-server")] +pub mod dist; -pub fn write_source(path: &Path, filename: &str, contents: &str) { - let p = path.join(filename); - let mut f = fs::File::create(p).unwrap(); - f.write_all(contents.as_bytes()).unwrap(); -} +pub const TC_CACHE_SIZE: u64 = 1024 * 1024 * 1024; // 1 gig pub fn init_cargo(path: &Path, cargo_name: &str) -> PathBuf { let cargo_path = path.join(cargo_name); @@ -129,559 +33,14 @@ pub fn prune_command(mut cmd: Command) -> Command { cmd } -pub fn sccache_command() -> Command { - prune_command(Command::new(assert_cmd::cargo::cargo_bin("sccache"))) -} - -pub fn cargo_command() -> Command { - prune_command(Command::new("cargo")) -} - -#[cfg(feature = "dist-server")] -pub fn sccache_dist_path() -> PathBuf { - assert_cmd::cargo::cargo_bin("sccache-dist") -} - -pub fn sccache_client_cfg( - tmpdir: &Path, - preprocessor_cache_mode: bool, -) -> sccache::config::FileConfig { - let cache_relpath = "client-cache"; - let dist_cache_relpath = "client-dist-cache"; - fs::create_dir(tmpdir.join(cache_relpath)).unwrap(); - fs::create_dir(tmpdir.join(dist_cache_relpath)).unwrap(); - - let disk_cache = sccache::config::DiskCacheConfig { - dir: tmpdir.join(cache_relpath), - preprocessor_cache_mode: sccache::config::PreprocessorCacheModeConfig { - use_preprocessor_cache_mode: preprocessor_cache_mode, - ..Default::default() - }, - ..Default::default() - }; - sccache::config::FileConfig { - cache: sccache::config::CacheConfigs { - azure: None, - disk: Some(disk_cache), - gcs: None, - gha: None, - memcached: None, - redis: None, - s3: None, - webdav: None, - oss: None, - }, - dist: sccache::config::DistConfig { - auth: Default::default(), // dangerously_insecure - scheduler_url: None, - cache_dir: tmpdir.join(dist_cache_relpath), - toolchains: vec![], - toolchain_cache_size: TC_CACHE_SIZE, - rewrite_includes_only: false, // TODO - }, - server_startup_timeout_ms: None, - } -} - -#[cfg(feature = "dist-server")] -fn sccache_scheduler_cfg() -> sccache::config::scheduler::Config { - sccache::config::scheduler::Config { - public_addr: SocketAddr::from(([0, 0, 0, 0], SCHEDULER_PORT)), - client_auth: sccache::config::scheduler::ClientAuth::Insecure, - server_auth: sccache::config::scheduler::ServerAuth::Token { - token: DIST_SERVER_TOKEN.to_owned(), - }, - } -} - -#[cfg(feature = "dist-server")] -fn sccache_server_cfg( - tmpdir: &Path, - scheduler_url: HTTPUrl, - server_ip: IpAddr, -) -> sccache::config::server::Config { - let relpath = "server-cache"; - fs::create_dir(tmpdir.join(relpath)).unwrap(); - - sccache::config::server::Config { - builder: sccache::config::server::BuilderType::Overlay { - build_dir: BUILD_DIR_CONTAINER_PATH.into(), - bwrap_path: DIST_IMAGE_BWRAP_PATH.into(), - }, - cache_dir: Path::new(CONFIGS_CONTAINER_PATH).join(relpath), - public_addr: SocketAddr::new(server_ip, SERVER_PORT), - bind_address: Some(SocketAddr::from(([0, 0, 0, 0], SERVER_PORT))), - scheduler_url, - scheduler_auth: sccache::config::server::SchedulerAuth::Token { - token: DIST_SERVER_TOKEN.to_owned(), - }, - toolchain_cache_size: TC_CACHE_SIZE, - } -} - -// TODO: this is copied from the sccache-dist binary - it's not clear where would be a better place to put the -// code so that it can be included here -#[cfg(feature = "dist-server")] -fn create_server_token(server_id: ServerId, auth_token: &str) -> String { - format!("{} {}", server_id.addr(), auth_token) -} - -#[cfg(feature = "dist-server")] -pub enum ServerHandle { - Container { cid: String, url: HTTPUrl }, - Process { pid: Pid, url: HTTPUrl }, -} - -#[cfg(feature = "dist-server")] -pub struct DistSystem { - sccache_dist: PathBuf, - tmpdir: PathBuf, - - scheduler_name: Option, - server_names: Vec, - server_pids: Vec, -} - -#[cfg(feature = "dist-server")] -impl DistSystem { - pub fn new(sccache_dist: &Path, tmpdir: &Path) -> Self { - // Make sure the docker image is available, building it if necessary - let mut child = Command::new("docker") - .args(["build", "-q", "-t", DIST_IMAGE, "-"]) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .unwrap(); - child - .stdin - .as_mut() - .unwrap() - .write_all(DIST_DOCKERFILE.as_bytes()) - .unwrap(); - let output = child.wait_with_output().unwrap(); - check_output(&output); - - let tmpdir = tmpdir.join("distsystem"); - fs::create_dir(&tmpdir).unwrap(); - - Self { - sccache_dist: sccache_dist.to_owned(), - tmpdir, - - scheduler_name: None, - server_names: vec![], - server_pids: vec![], - } - } - - pub fn add_scheduler(&mut self) { - let scheduler_cfg_relpath = "scheduler-cfg.json"; - let scheduler_cfg_path = self.tmpdir.join(scheduler_cfg_relpath); - let scheduler_cfg_container_path = - Path::new(CONFIGS_CONTAINER_PATH).join(scheduler_cfg_relpath); - let scheduler_cfg = sccache_scheduler_cfg(); - fs::File::create(scheduler_cfg_path) - .unwrap() - .write_all(&serde_json::to_vec(&scheduler_cfg).unwrap()) - .unwrap(); - - // Create the scheduler - let scheduler_name = make_container_name("scheduler"); - let output = Command::new("docker") - .args([ - "run", - "--name", - &scheduler_name, - "-e", - "SCCACHE_NO_DAEMON=1", - "-e", - "SCCACHE_LOG=debug", - "-e", - "RUST_BACKTRACE=1", - "--network", - "host", - "-v", - &format!("{}:/sccache-dist:z", self.sccache_dist.to_str().unwrap()), - "-v", - &format!( - "{}:{}:z", - self.tmpdir.to_str().unwrap(), - CONFIGS_CONTAINER_PATH - ), - "-d", - DIST_IMAGE, - "bash", - "-c", - &format!( - r#" - set -o errexit && - exec /sccache-dist scheduler --config {cfg} - "#, - cfg = scheduler_cfg_container_path.to_str().unwrap() - ), - ]) - .output() - .unwrap(); - self.scheduler_name = Some(scheduler_name); - - check_output(&output); - - let scheduler_url = self.scheduler_url(); - wait_for_http(scheduler_url, Duration::from_millis(100), MAX_STARTUP_WAIT); - wait_for( - || { - let status = self.scheduler_status(); - if matches!( - status, - SchedulerStatusResult { - num_servers: 0, - num_cpus: _, - in_progress: 0 - } - ) { - Ok(()) - } else { - Err(format!("{:?}", status)) - } - }, - Duration::from_millis(100), - MAX_STARTUP_WAIT, - ); - } - - pub fn add_server(&mut self) -> ServerHandle { - let server_cfg_relpath = format!("server-cfg-{}.json", self.server_names.len()); - let server_cfg_path = self.tmpdir.join(&server_cfg_relpath); - let server_cfg_container_path = Path::new(CONFIGS_CONTAINER_PATH).join(server_cfg_relpath); - - let server_name = make_container_name("server"); - let output = Command::new("docker") - .args([ - "run", - // Important for the bubblewrap builder - "--privileged", - "--name", - &server_name, - "-e", - "SCCACHE_LOG=debug", - "-e", - "RUST_BACKTRACE=1", - "--network", - "host", - "-v", - &format!("{}:/sccache-dist:z", self.sccache_dist.to_str().unwrap()), - "-v", - &format!( - "{}:{}:z", - self.tmpdir.to_str().unwrap(), - CONFIGS_CONTAINER_PATH - ), - "-d", - DIST_IMAGE, - "bash", - "-c", - &format!( - r#" - set -o errexit && - while [ ! -f {cfg}.ready ]; do sleep 0.1; done && - exec /sccache-dist server --config {cfg} - "#, - cfg = server_cfg_container_path.to_str().unwrap() - ), - ]) - .output() - .unwrap(); - self.server_names.push(server_name.clone()); - - check_output(&output); - - let server_ip = IpAddr::from_str("127.0.0.1").unwrap(); - let server_cfg = sccache_server_cfg(&self.tmpdir, self.scheduler_url(), server_ip); - fs::File::create(&server_cfg_path) - .unwrap() - .write_all(&serde_json::to_vec(&server_cfg).unwrap()) - .unwrap(); - fs::File::create(format!("{}.ready", server_cfg_path.to_str().unwrap())).unwrap(); - - let url = HTTPUrl::from_url( - reqwest::Url::parse(&format!("https://{}:{}", server_ip, SERVER_PORT)).unwrap(), - ); - let handle = ServerHandle::Container { - cid: server_name, - url, - }; - self.wait_server_ready(&handle); - handle - } - - pub fn add_custom_server( - &mut self, - handler: S, - ) -> ServerHandle { - let server_addr = { - let ip = IpAddr::from_str("127.0.0.1").unwrap(); - let listener = net::TcpListener::bind(SocketAddr::from((ip, 0))).unwrap(); - listener.local_addr().unwrap() - }; - let token = create_server_token(ServerId::new(server_addr), DIST_SERVER_TOKEN); - let server = dist::http::Server::new( - server_addr, - Some(SocketAddr::from(([0, 0, 0, 0], server_addr.port()))), - self.scheduler_url().to_url(), - token, - handler, - ) - .unwrap(); - let pid = match unsafe { nix::unistd::fork() }.unwrap() { - ForkResult::Parent { child } => { - self.server_pids.push(child); - child - } - ForkResult::Child => { - env::set_var("SCCACHE_LOG", "sccache=trace"); - env_logger::try_init().unwrap(); - server.start().unwrap(); - unreachable!(); - } - }; - - let url = - HTTPUrl::from_url(reqwest::Url::parse(&format!("https://{}", server_addr)).unwrap()); - let handle = ServerHandle::Process { pid, url }; - self.wait_server_ready(&handle); - handle - } - - pub fn restart_server(&mut self, handle: &ServerHandle) { - match handle { - ServerHandle::Container { cid, url: _ } => { - let output = Command::new("docker") - .args(["restart", cid]) - .output() - .unwrap(); - check_output(&output); - } - ServerHandle::Process { pid: _, url: _ } => { - // TODO: pretty easy, just no need yet - panic!("restart not yet implemented for pids") - } - } - self.wait_server_ready(handle) - } - - pub fn wait_server_ready(&mut self, handle: &ServerHandle) { - let url = match handle { - ServerHandle::Container { cid: _, url } | ServerHandle::Process { pid: _, url } => { - url.clone() - } - }; - wait_for_http(url, Duration::from_millis(100), MAX_STARTUP_WAIT); - wait_for( - || { - let status = self.scheduler_status(); - if matches!( - status, - SchedulerStatusResult { - num_servers: 1, - num_cpus: _, - in_progress: 0 - } - ) { - Ok(()) - } else { - Err(format!("{:?}", status)) - } - }, - Duration::from_millis(100), - MAX_STARTUP_WAIT, - ); - } - - pub fn scheduler_url(&self) -> HTTPUrl { - let url = format!("http://127.0.0.1:{}", SCHEDULER_PORT); - HTTPUrl::from_url(reqwest::Url::parse(&url).unwrap()) - } - - fn scheduler_status(&self) -> SchedulerStatusResult { - let res = reqwest::blocking::get(dist::http::urls::scheduler_status( - &self.scheduler_url().to_url(), - )) - .unwrap(); - assert!(res.status().is_success()); - bincode::deserialize_from(res).unwrap() - } -} - -// If you want containers to hang around (e.g. for debugging), comment out the "rm -f" lines -#[cfg(feature = "dist-server")] -impl Drop for DistSystem { - fn drop(&mut self) { - let mut did_err = false; - - // Panicking halfway through drop would either abort (if it's a double panic) or leave us with - // resources that aren't yet cleaned up. Instead, do as much as possible then decide what to do - // at the end - panic (if not already doing so) or let the panic continue - macro_rules! droperr { - ($e:expr) => { - match $e { - Ok(()) => (), - Err(e) => { - did_err = true; - eprintln!("Error with {}: {}", stringify!($e), e) - } - } - }; - } - - let mut logs = vec![]; - let mut outputs = vec![]; - let mut exits = vec![]; - - if let Some(scheduler_name) = self.scheduler_name.as_ref() { - droperr!(Command::new("docker") - .args(["logs", scheduler_name]) - .output() - .map(|o| logs.push((scheduler_name, o)))); - droperr!(Command::new("docker") - .args(["kill", scheduler_name]) - .output() - .map(|o| outputs.push((scheduler_name, o)))); - droperr!(Command::new("docker") - .args(["rm", "-f", scheduler_name]) - .output() - .map(|o| outputs.push((scheduler_name, o)))); - } - for server_name in self.server_names.iter() { - droperr!(Command::new("docker") - .args(["logs", server_name]) - .output() - .map(|o| logs.push((server_name, o)))); - droperr!(Command::new("docker") - .args(["kill", server_name]) - .output() - .map(|o| outputs.push((server_name, o)))); - droperr!(Command::new("docker") - .args(["rm", "-f", server_name]) - .output() - .map(|o| outputs.push((server_name, o)))); - } - for &pid in self.server_pids.iter() { - droperr!(nix::sys::signal::kill(pid, Signal::SIGINT)); - thread::sleep(Duration::from_millis(100)); - let mut killagain = true; // Default to trying to kill again, e.g. if there was an error waiting on the pid - droperr!( - nix::sys::wait::waitpid(pid, Some(WaitPidFlag::WNOHANG)).map(|ws| { - if ws != WaitStatus::StillAlive { - killagain = false; - exits.push(ws) - } - }) - ); - if killagain { - eprintln!("SIGINT didn't kill process, trying SIGKILL"); - droperr!(nix::sys::signal::kill(pid, Signal::SIGKILL)); - droperr!(nix::sys::wait::waitpid(pid, Some(WaitPidFlag::WNOHANG)) - .map_err(|e| e.to_string()) - .and_then(|ws| if ws == WaitStatus::StillAlive { - Err("process alive after sigkill".to_owned()) - } else { - exits.push(ws); - Ok(()) - })); - } - } - - for ( - container, - Output { - status, - stdout, - stderr, - }, - ) in logs - { - println!( - "LOGS == ({}) ==\n> {} <:\n## STDOUT\n{}\n\n## STDERR\n{}\n====", - status, - container, - String::from_utf8_lossy(&stdout), - String::from_utf8_lossy(&stderr) - ); - } - for ( - container, - Output { - status, - stdout, - stderr, - }, - ) in outputs - { - println!( - "OUTPUTS == ({}) ==\n> {} <:\n## STDOUT\n{}\n\n## STDERR\n{}\n====", - status, - container, - String::from_utf8_lossy(&stdout), - String::from_utf8_lossy(&stderr) - ); - } - for exit in exits { - println!("EXIT: {:?}", exit) - } - - if did_err && !thread::panicking() { - panic!("Encountered failures during dist system teardown") - } - } -} - -fn make_container_name(tag: &str) -> String { - format!( - "{}_{}_{}", - CONTAINER_NAME_PREFIX, - tag, - Uuid::new_v4().hyphenated() - ) -} - -fn check_output(output: &Output) { - if !output.status.success() { - println!("{}\n\n[BEGIN STDOUT]\n===========\n{}\n===========\n[FIN STDOUT]\n\n[BEGIN STDERR]\n===========\n{}\n===========\n[FIN STDERR]\n\n", - output.status, String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr)); - panic!() - } -} - -#[cfg(feature = "dist-server")] -fn wait_for_http(url: HTTPUrl, interval: Duration, max_wait: Duration) { - // TODO: after upgrading to reqwest >= 0.9, use 'danger_accept_invalid_certs' and stick with that rather than tcp - wait_for( - || { - let url = url.to_url(); - let url = url.socket_addrs(|| None).unwrap(); - match net::TcpStream::connect(url.as_slice()) { - Ok(_) => Ok(()), - Err(e) => Err(e.to_string()), - } - }, - interval, - max_wait, - ) +pub fn write_json_cfg(path: &Path, filename: &str, contents: &T) { + let p = path.join(filename); + let mut f = fs::File::create(p).unwrap(); + f.write_all(&serde_json::to_vec(contents).unwrap()).unwrap(); } -fn wait_for Result<(), String>>(f: F, interval: Duration, max_wait: Duration) { - let start = Instant::now(); - let mut lasterr; - loop { - match f() { - Ok(()) => return, - Err(e) => lasterr = e, - } - if start.elapsed() > max_wait { - break; - } - thread::sleep(interval) - } - panic!("wait timed out, last error result: {}", lasterr) +pub fn write_source(path: &Path, filename: &str, contents: &str) { + let p = path.join(filename); + let mut f = fs::File::create(p).unwrap(); + f.write_all(contents.as_bytes()).unwrap(); } diff --git a/tests/system.rs b/tests/system.rs index d4ef4fd63..2a9795e95 100644 --- a/tests/system.rs +++ b/tests/system.rs @@ -20,8 +20,8 @@ #[macro_use] extern crate log; use crate::harness::{ - get_stats, sccache_client_cfg, sccache_command, start_local_daemon, stop_local_daemon, - write_json_cfg, write_source, zero_stats, + client::{sccache_client_cfg, SccacheClient}, + write_json_cfg, write_source, }; use assert_cmd::prelude::*; use fs::File; @@ -29,6 +29,10 @@ use fs_err as fs; use log::Level::Trace; use predicates::prelude::*; use regex::Regex; +use sccache::{ + compiler::{CCompilerKind, CompilerKind, Language}, + server::ServerStats, +}; use serial_test::serial; use std::collections::HashMap; use std::env; @@ -119,9 +123,9 @@ fn compile_cuda_cmdline>( compile_flag: &str, input: &str, output: &str, - mut extra_args: Vec, + extra_args: &[OsString], ) -> Vec { - let mut arg = match compiler { + let mut args = match compiler { "nvcc" => vec_from!(OsString, exe.as_ref(), compile_flag, input, "-o", output), "clang++" => { vec_from!( @@ -147,9 +151,12 @@ fn compile_cuda_cmdline>( _ => panic!("Unsupported compiler: {}", compiler), }; if !extra_args.is_empty() { - arg.append(&mut extra_args) + args.append(&mut extra_args.to_vec()) } - arg + args.iter() + .filter(|x| !x.is_empty()) + .cloned() + .collect::>() } // TODO: This will fail if gcc/clang is actually a ccache wrapper, as it is the @@ -211,7 +218,7 @@ fn copy_to_tempdir(inputs: &[&str], tempdir: &Path) { } } -fn test_basic_compile(compiler: Compiler, tempdir: &Path) { +fn test_basic_compile(client: &SccacheClient, compiler: Compiler, tempdir: &Path) { let Compiler { name, exe, @@ -223,7 +230,8 @@ fn test_basic_compile(compiler: Compiler, tempdir: &Path) { let out_file = tempdir.join(OUTPUT); trace!("compile"); - sccache_command() + client + .cmd() .args(compile_cmdline(name, &exe, INPUT, OUTPUT, Vec::new())) .current_dir(tempdir) .envs(env_vars.clone()) @@ -231,18 +239,18 @@ fn test_basic_compile(compiler: Compiler, tempdir: &Path) { .success(); assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); trace!("request stats"); - get_stats(|info| { - assert_eq!(1, info.stats.compile_requests); - assert_eq!(1, info.stats.requests_executed); - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_misses.get("C/C++").unwrap()); - let adv_key = adv_key_kind("c", compiler.name); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_key).unwrap()); - }); + let stats = client.stats().unwrap(); + assert_eq!(1, stats.compile_requests); + assert_eq!(1, stats.requests_executed); + assert_eq!(0, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); + assert_eq!(&1, stats.cache_misses.get("C/C++").unwrap()); + let adv_key = adv_key_kind("c", compiler.name); + assert_eq!(&1, stats.cache_misses.get_adv(&adv_key).unwrap()); trace!("compile"); fs::remove_file(&out_file).unwrap(); - sccache_command() + client + .cmd() .args(compile_cmdline(name, &exe, INPUT, OUTPUT, Vec::new())) .current_dir(tempdir) .envs(env_vars) @@ -250,20 +258,19 @@ fn test_basic_compile(compiler: Compiler, tempdir: &Path) { .success(); assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); trace!("request stats"); - get_stats(|info| { - assert_eq!(2, info.stats.compile_requests); - assert_eq!(2, info.stats.requests_executed); - assert_eq!(1, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("C/C++").unwrap()); - assert_eq!(&1, info.stats.cache_misses.get("C/C++").unwrap()); - let adv_key = adv_key_kind("c", compiler.name); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_key).unwrap()); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_key).unwrap()); - }); + let stats = client.stats().unwrap(); + assert_eq!(2, stats.compile_requests); + assert_eq!(2, stats.requests_executed); + assert_eq!(1, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); + assert_eq!(&1, stats.cache_hits.get("C/C++").unwrap()); + assert_eq!(&1, stats.cache_misses.get("C/C++").unwrap()); + let adv_key = adv_key_kind("c", compiler.name); + assert_eq!(&1, stats.cache_hits.get_adv(&adv_key).unwrap()); + assert_eq!(&1, stats.cache_misses.get_adv(&adv_key).unwrap()); } -fn test_noncacheable_stats(compiler: Compiler, tempdir: &Path) { +fn test_noncacheable_stats(client: &SccacheClient, compiler: Compiler, tempdir: &Path) { let Compiler { name, exe, @@ -273,7 +280,8 @@ fn test_noncacheable_stats(compiler: Compiler, tempdir: &Path) { copy_to_tempdir(&[INPUT], tempdir); trace!("compile"); - sccache_command() + client + .cmd() .arg(&exe) .arg("-E") .arg(INPUT) @@ -282,15 +290,14 @@ fn test_noncacheable_stats(compiler: Compiler, tempdir: &Path) { .assert() .success(); trace!("request stats"); - get_stats(|info| { - assert_eq!(1, info.stats.compile_requests); - assert_eq!(0, info.stats.requests_executed); - assert_eq!(1, info.stats.not_cached.len()); - assert_eq!(Some(&1), info.stats.not_cached.get("-E")); - }); + let stats = client.stats().unwrap(); + assert_eq!(1, stats.compile_requests); + assert_eq!(0, stats.requests_executed); + assert_eq!(1, stats.not_cached.len()); + assert_eq!(Some(&1), stats.not_cached.get("-E")); } -fn test_msvc_deps(compiler: Compiler, tempdir: &Path) { +fn test_msvc_deps(client: &SccacheClient, compiler: Compiler, tempdir: &Path) { let Compiler { name, exe, @@ -300,7 +307,8 @@ fn test_msvc_deps(compiler: Compiler, tempdir: &Path) { trace!("compile with /sourceDependencies"); let mut args = compile_cmdline(name, exe, INPUT, OUTPUT, Vec::new()); args.push("/sourceDependenciestest.o.json".into()); - sccache_command() + client + .cmd() .args(&args) .current_dir(tempdir) .envs(env_vars) @@ -321,7 +329,7 @@ fn test_msvc_deps(compiler: Compiler, tempdir: &Path) { assert_ne!(includes.len(), 0); } -fn test_msvc_responsefile(compiler: Compiler, tempdir: &Path) { +fn test_msvc_responsefile(client: &SccacheClient, compiler: Compiler, tempdir: &Path) { let Compiler { name: _, exe, @@ -337,7 +345,8 @@ fn test_msvc_responsefile(compiler: Compiler, tempdir: &Path) { } let args = vec_from!(OsString, exe, &format!("@{cmd_file_name}")); - sccache_command() + client + .cmd() .args(&args) .current_dir(tempdir) .envs(env_vars) @@ -348,7 +357,7 @@ fn test_msvc_responsefile(compiler: Compiler, tempdir: &Path) { fs::remove_file(&out_file).unwrap(); } -fn test_gcc_mp_werror(compiler: Compiler, tempdir: &Path) { +fn test_gcc_mp_werror(client: &SccacheClient, compiler: Compiler, tempdir: &Path) { let Compiler { name, exe, @@ -360,7 +369,8 @@ fn test_gcc_mp_werror(compiler: Compiler, tempdir: &Path) { OsString, "-MD", "-MP", "-MF", "foo.pp", "-Werror" )); // This should fail, but the error should be from the #error! - sccache_command() + client + .cmd() .args(&args) .current_dir(tempdir) .envs(env_vars) @@ -373,14 +383,18 @@ fn test_gcc_mp_werror(compiler: Compiler, tempdir: &Path) { ); } -fn test_gcc_fprofile_generate_source_changes(compiler: Compiler, tempdir: &Path) { +fn test_gcc_fprofile_generate_source_changes( + client: &SccacheClient, + compiler: Compiler, + tempdir: &Path, +) { let Compiler { name, exe, env_vars, } = compiler; trace!("test -fprofile-generate with different source inputs"); - zero_stats(); + client.zero_stats(); const SRC: &str = "source.c"; write_source( tempdir, @@ -398,31 +412,31 @@ int main(int argc, char** argv) { let mut args = compile_cmdline(name, exe, SRC, OUTPUT, Vec::new()); args.extend(vec_from!(OsString, "-fprofile-generate")); trace!("compile source.c (1)"); - sccache_command() + client + .cmd() .args(&args) .current_dir(tempdir) .envs(env_vars.clone()) .assert() .success(); - get_stats(|info| { - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_misses.get("C/C++").unwrap()); - }); + let stats = client.stats().unwrap(); + assert_eq!(0, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); + assert_eq!(&1, stats.cache_misses.get("C/C++").unwrap()); // Compile the same source again to ensure we can get a cache hit. trace!("compile source.c (2)"); - sccache_command() + client + .cmd() .args(&args) .current_dir(tempdir) .envs(env_vars.clone()) .assert() .success(); - get_stats(|info| { - assert_eq!(1, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("C/C++").unwrap()); - assert_eq!(&1, info.stats.cache_misses.get("C/C++").unwrap()); - }); + let stats = client.stats().unwrap(); + assert_eq!(1, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); + assert_eq!(&1, stats.cache_hits.get("C/C++").unwrap()); + assert_eq!(&1, stats.cache_misses.get("C/C++").unwrap()); // Now write out a slightly different source file that will preprocess to the same thing, // modulo line numbers. This should not be a cache hit because line numbers are important // with -fprofile-generate. @@ -441,18 +455,18 @@ int main(int argc, char** argv) { ", ); trace!("compile source.c (3)"); - sccache_command() + client + .cmd() .args(&args) .current_dir(tempdir) .envs(env_vars) .assert() .success(); - get_stats(|info| { - assert_eq!(1, info.stats.cache_hits.all()); - assert_eq!(2, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("C/C++").unwrap()); - assert_eq!(&2, info.stats.cache_misses.get("C/C++").unwrap()); - }); + let stats = client.stats().unwrap(); + assert_eq!(1, stats.cache_hits.all()); + assert_eq!(2, stats.cache_misses.all()); + assert_eq!(&1, stats.cache_hits.get("C/C++").unwrap()); + assert_eq!(&2, stats.cache_misses.get("C/C++").unwrap()); } /* test case like this: @@ -463,66 +477,74 @@ int main(int argc, char** argv) { sccache g++ -c -g -gsplit-dwarf test.cc -o test2.o --- > cache miss strings test2.o |grep test2.dwo */ -fn test_split_dwarf_object_generate_output_dir_changes(compiler: Compiler, tempdir: &Path) { +fn test_split_dwarf_object_generate_output_dir_changes( + client: &SccacheClient, + compiler: Compiler, + tempdir: &Path, +) { let Compiler { name, exe, env_vars, } = compiler; trace!("test -g -gsplit-dwarf with different output"); - zero_stats(); + client.zero_stats(); const SRC: &str = "source.c"; write_source(tempdir, SRC, "int test(){}"); let mut args = compile_cmdline(name, exe.clone(), SRC, "test1.o", Vec::new()); args.extend(vec_from!(OsString, "-g")); args.extend(vec_from!(OsString, "-gsplit-dwarf")); trace!("compile source.c (1)"); - sccache_command() + client + .cmd() .args(&args) .current_dir(tempdir) .envs(env_vars.clone()) .assert() .success(); - get_stats(|info| { - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_misses.get("C/C++").unwrap()); - }); + let stats = client.stats().unwrap(); + assert_eq!(0, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); + assert_eq!(&1, stats.cache_misses.get("C/C++").unwrap()); // Compile the same source again to ensure we can get a cache hit. trace!("compile source.c (2)"); - sccache_command() + client + .cmd() .args(&args) .current_dir(tempdir) .envs(env_vars.clone()) .assert() .success(); - get_stats(|info| { - assert_eq!(1, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("C/C++").unwrap()); - assert_eq!(&1, info.stats.cache_misses.get("C/C++").unwrap()); - }); + let stats = client.stats().unwrap(); + assert_eq!(1, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); + assert_eq!(&1, stats.cache_hits.get("C/C++").unwrap()); + assert_eq!(&1, stats.cache_misses.get("C/C++").unwrap()); // Compile the same source again with different output // to ensure we can force generate new object file. let mut args2 = compile_cmdline(name, exe, SRC, "test2.o", Vec::new()); args2.extend(vec_from!(OsString, "-g")); args2.extend(vec_from!(OsString, "-gsplit-dwarf")); trace!("compile source.c (2)"); - sccache_command() + client + .cmd() .args(&args2) .current_dir(tempdir) .envs(env_vars.clone()) .assert() .success(); - get_stats(|info| { - assert_eq!(1, info.stats.cache_hits.all()); - assert_eq!(2, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("C/C++").unwrap()); - assert_eq!(&2, info.stats.cache_misses.get("C/C++").unwrap()); - }); + let stats = client.stats().unwrap(); + assert_eq!(1, stats.cache_hits.all()); + assert_eq!(2, stats.cache_misses.all()); + assert_eq!(&1, stats.cache_hits.get("C/C++").unwrap()); + assert_eq!(&2, stats.cache_misses.get("C/C++").unwrap()); } -fn test_gcc_clang_no_warnings_from_macro_expansion(compiler: Compiler, tempdir: &Path) { +fn test_gcc_clang_no_warnings_from_macro_expansion( + client: &SccacheClient, + compiler: Compiler, + tempdir: &Path, +) { let Compiler { name, exe, @@ -533,7 +555,8 @@ fn test_gcc_clang_no_warnings_from_macro_expansion(compiler: Compiler, tempdir: copy_to_tempdir(&[INPUT_MACRO_EXPANSION], tempdir); trace!("compile"); - sccache_command() + client + .cmd() .args( [ &compile_cmdline(name, exe, INPUT_MACRO_EXPANSION, OUTPUT, Vec::new())[..], @@ -548,7 +571,7 @@ fn test_gcc_clang_no_warnings_from_macro_expansion(compiler: Compiler, tempdir: .stderr(predicates::str::contains("warning:").from_utf8().not()); } -fn test_compile_with_define(compiler: Compiler, tempdir: &Path) { +fn test_compile_with_define(client: &SccacheClient, compiler: Compiler, tempdir: &Path) { let Compiler { name, exe, @@ -559,7 +582,8 @@ fn test_compile_with_define(compiler: Compiler, tempdir: &Path) { copy_to_tempdir(&[INPUT_WITH_DEFINE], tempdir); trace!("compile"); - sccache_command() + client + .cmd() .args( [ &compile_cmdline(name, exe, INPUT_WITH_DEFINE, OUTPUT, Vec::new())[..], @@ -574,7 +598,7 @@ fn test_compile_with_define(compiler: Compiler, tempdir: &Path) { .stderr(predicates::str::contains("warning:").from_utf8().not()); } -fn test_gcc_clang_depfile(compiler: Compiler, tempdir: &Path) { +fn test_gcc_clang_depfile(client: &SccacheClient, compiler: Compiler, tempdir: &Path) { let Compiler { name, exe, @@ -585,7 +609,8 @@ fn test_gcc_clang_depfile(compiler: Compiler, tempdir: &Path) { fs::copy(tempdir.join(INPUT), tempdir.join("same-content.c")).unwrap(); trace!("compile"); - sccache_command() + client + .cmd() .args(compile_cmdline( name, exe.clone(), @@ -598,7 +623,8 @@ fn test_gcc_clang_depfile(compiler: Compiler, tempdir: &Path) { .envs(env_vars.clone()) .assert() .success(); - sccache_command() + client + .cmd() .args(compile_cmdline( name, exe, @@ -624,26 +650,31 @@ fn test_gcc_clang_depfile(compiler: Compiler, tempdir: &Path) { assert_ne!(first, second); } -fn run_sccache_command_tests(compiler: Compiler, tempdir: &Path, preprocessor_cache_mode: bool) { +fn run_sccache_command_tests( + client: &SccacheClient, + compiler: Compiler, + tempdir: &Path, + preprocessor_cache_mode: bool, +) { if compiler.name != "clang++" { - test_basic_compile(compiler.clone(), tempdir); + test_basic_compile(client, compiler.clone(), tempdir); } - test_compile_with_define(compiler.clone(), tempdir); + test_compile_with_define(client, compiler.clone(), tempdir); if compiler.name == "cl.exe" { - test_msvc_deps(compiler.clone(), tempdir); - test_msvc_responsefile(compiler.clone(), tempdir); + test_msvc_deps(client, compiler.clone(), tempdir); + test_msvc_responsefile(client, compiler.clone(), tempdir); } if compiler.name == "gcc" { - test_gcc_mp_werror(compiler.clone(), tempdir); - test_gcc_fprofile_generate_source_changes(compiler.clone(), tempdir); + test_gcc_mp_werror(client, compiler.clone(), tempdir); + test_gcc_fprofile_generate_source_changes(client, compiler.clone(), tempdir); } if compiler.name == "clang" || compiler.name == "gcc" { - test_gcc_clang_no_warnings_from_macro_expansion(compiler.clone(), tempdir); - test_split_dwarf_object_generate_output_dir_changes(compiler.clone(), tempdir); - test_gcc_clang_depfile(compiler.clone(), tempdir); + test_gcc_clang_no_warnings_from_macro_expansion(client, compiler.clone(), tempdir); + test_split_dwarf_object_generate_output_dir_changes(client, compiler.clone(), tempdir); + test_gcc_clang_depfile(client, compiler.clone(), tempdir); } if compiler.name == "clang++" { - test_clang_multicall(compiler.clone(), tempdir); + test_clang_multicall(client, compiler.clone(), tempdir); } // If we are testing with clang-14 or later, we expect the -fminimize-whitespace flag to be used. @@ -672,6 +703,7 @@ fn run_sccache_command_tests(compiler: Compiler, tempdir: &Path, preprocessor_ca ), }; test_clang_cache_whitespace_normalization( + client, compiler, tempdir, !is_appleclang && major >= 14, @@ -679,6 +711,7 @@ fn run_sccache_command_tests(compiler: Compiler, tempdir: &Path, preprocessor_ca ); } else { test_clang_cache_whitespace_normalization( + client, compiler, tempdir, false, @@ -687,7 +720,22 @@ fn run_sccache_command_tests(compiler: Compiler, tempdir: &Path, preprocessor_ca } } -fn test_nvcc_cuda_compiles(compiler: &Compiler, tempdir: &Path) { +#[derive(Clone, Debug, Default)] +struct AdditionalStats { + cache_writes: Option, + compilations: Option, + compile_requests: Option, + requests_executed: Option, + requests_not_compile: Option, + cache_hits: Option>, + cache_misses: Option>, +} + +fn test_nvcc_cuda_compiles(client: &SccacheClient, compiler: &Compiler, tempdir: &Path) { + let mut stats = client.stats().unwrap(); + + let extra_args = vec![]; + let Compiler { name, exe, @@ -697,269 +745,176 @@ fn test_nvcc_cuda_compiles(compiler: &Compiler, tempdir: &Path) { // Compile multiple source files. copy_to_tempdir(&[INPUT_FOR_CUDA_A, INPUT_FOR_CUDA_B], tempdir); - let out_file = tempdir.join(OUTPUT); - trace!("compile A"); - sccache_command() - .args(compile_cuda_cmdline( - name, - exe, - "-c", - // relative path for input - INPUT_FOR_CUDA_A, - // relative path for output - out_file.file_name().unwrap().to_string_lossy().as_ref(), - Vec::new(), - )) - .current_dir(tempdir) - .envs(env_vars.clone()) - .assert() - .success(); - assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); - fs::remove_file(&out_file).unwrap(); - trace!("compile A request stats"); - get_stats(|info| { - assert_eq!(1, info.stats.compile_requests); - assert_eq!(5, info.stats.requests_executed); - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(4, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_misses.get("CUDA").unwrap()); - assert_eq!( - &1, - info.stats.cache_misses.get("CUDA (Device code)").unwrap() - ); - assert_eq!(&1, info.stats.cache_misses.get("PTX").unwrap()); - assert_eq!(&1, info.stats.cache_misses.get("CUBIN").unwrap()); - assert!(info.stats.cache_misses.get("C/C++").is_none()); - let adv_cuda_key = adv_key_kind("cuda", compiler.name); - let adv_cudafe_key = adv_key_kind("cudafe++", compiler.name); - let adv_ptx_key = adv_key_kind("ptx", compiler.name); - let adv_cubin_key = adv_key_kind("cubin", compiler.name); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_cuda_key).unwrap()); + let build_dir = PathBuf::from("build"); + fs::create_dir_all(tempdir.join(&build_dir)).unwrap(); + + let mut run_cuda_test = |compile_flag: &str, + input: &Path, + output: &Path, + extra_args: &[OsString], + additional_stats: AdditionalStats| { + client + .cmd() + .args(compile_cuda_cmdline( + name, + exe, + compile_flag, + input.to_string_lossy().as_ref(), + output.to_string_lossy().as_ref(), + extra_args, + )) + .current_dir(tempdir) + .envs(env_vars.clone()) + .assert() + .success(); + + assert!(fs::metadata(tempdir.join(output)) + .map(|m| m.len() > 0) + .unwrap()); + + fs::remove_file(tempdir.join(output)).unwrap(); + + stats.cache_writes += additional_stats.cache_writes.unwrap_or(0); + stats.compilations += additional_stats.compilations.unwrap_or(0); + stats.compile_requests += additional_stats.compile_requests.unwrap_or(0); + stats.requests_executed += additional_stats.requests_executed.unwrap_or(0); + stats.requests_not_compile += additional_stats.requests_not_compile.unwrap_or(0); + stats.non_cacheable_compilations += 1; + + for (kind, lang, count) in additional_stats.cache_hits.unwrap_or_default() { + let kind = CompilerKind::C(kind); + for _ in 0..count { + stats.cache_hits.increment(&kind, &lang); + } + } + + for (kind, lang, count) in additional_stats.cache_misses.unwrap_or_default() { + let kind = CompilerKind::C(kind); + for _ in 0..count { + stats.cache_misses.increment(&kind, &lang); + } + } + assert_eq!( - &1, - info.stats.cache_misses.get_adv(&adv_cudafe_key).unwrap() + stats, + ServerStats { + // TODO: Fix this in the next PR + cache_errors: stats.cache_errors.clone(), + cache_write_duration: stats.cache_write_duration, + cache_read_hit_duration: stats.cache_read_hit_duration, + compiler_write_duration: stats.compiler_write_duration, + ..client.stats().unwrap() + } ); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_ptx_key).unwrap()); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_cubin_key).unwrap()); - }); + }; trace!("compile A"); - sccache_command() - .args(compile_cuda_cmdline( - name, - exe, - "-c", - // relative path for input - INPUT_FOR_CUDA_A, - // absolute path for output - out_file.to_string_lossy().as_ref(), - Vec::new(), - )) - .current_dir(tempdir) - .envs(env_vars.clone()) - .assert() - .success(); - assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); - fs::remove_file(&out_file).unwrap(); - trace!("compile A request stats"); - get_stats(|info| { - assert_eq!(2, info.stats.compile_requests); - assert_eq!(10, info.stats.requests_executed); - assert_eq!(4, info.stats.cache_hits.all()); - assert_eq!(4, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA").unwrap()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA (Device code)").unwrap()); - assert_eq!(&1, info.stats.cache_hits.get("PTX").unwrap()); - assert_eq!(&1, info.stats.cache_hits.get("CUBIN").unwrap()); - assert_eq!(&1, info.stats.cache_misses.get("CUDA").unwrap()); - assert_eq!( - &1, - info.stats.cache_misses.get("CUDA (Device code)").unwrap() - ); - assert_eq!(&1, info.stats.cache_misses.get("PTX").unwrap()); - assert_eq!(&1, info.stats.cache_misses.get("CUBIN").unwrap()); - assert!(info.stats.cache_misses.get("C/C++").is_none()); - let adv_cuda_key = adv_key_kind("cuda", compiler.name); - let adv_cudafe_key = adv_key_kind("cudafe++", compiler.name); - let adv_ptx_key = adv_key_kind("ptx", compiler.name); - let adv_cubin_key = adv_key_kind("cubin", compiler.name); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_cuda_key).unwrap()); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_cudafe_key).unwrap()); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_ptx_key).unwrap()); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_cubin_key).unwrap()); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_cuda_key).unwrap()); - assert_eq!( - &1, - info.stats.cache_misses.get_adv(&adv_cudafe_key).unwrap() - ); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_ptx_key).unwrap()); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_cubin_key).unwrap()); - }); + run_cuda_test( + "-c", + Path::new(INPUT_FOR_CUDA_A), // relative path for input + &build_dir.join(OUTPUT), // relative path for output + &extra_args, + AdditionalStats { + cache_writes: Some(4), + compilations: Some(5), + compile_requests: Some(1), + requests_executed: Some(5), + cache_misses: Some(vec![ + (CCompilerKind::CudaFE, Language::CudaFE, 1), + (CCompilerKind::Cicc, Language::Ptx, 1), + (CCompilerKind::Ptxas, Language::Cubin, 1), + (CCompilerKind::Nvcc, Language::Cuda, 1), + ]), + ..Default::default() + }, + ); + + trace!("compile A"); + run_cuda_test( + "-c", + Path::new(INPUT_FOR_CUDA_A), // relative path for input + &tempdir.join(&build_dir).join(OUTPUT), // absolute path for output + &extra_args, + AdditionalStats { + cache_writes: Some(0), + compilations: Some(1), + compile_requests: Some(1), + requests_executed: Some(5), + cache_hits: Some(vec![ + (CCompilerKind::CudaFE, Language::CudaFE, 1), + (CCompilerKind::Cicc, Language::Ptx, 1), + (CCompilerKind::Ptxas, Language::Cubin, 1), + (CCompilerKind::Nvcc, Language::Cuda, 1), + ]), + ..Default::default() + }, + ); // By compiling another input source we verify that the pre-processor // phase is correctly running and outputting text trace!("compile B"); - sccache_command() - .args(compile_cuda_cmdline( - name, - exe, - "-c", - // absolute path for input - &tempdir.join(INPUT_FOR_CUDA_B).to_string_lossy(), - // absolute path for output - out_file.to_string_lossy().as_ref(), - Vec::new(), - )) - .current_dir(tempdir) - .envs(env_vars.clone()) - .assert() - .success(); - assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); - fs::remove_file(&out_file).unwrap(); - trace!("compile B request stats"); - get_stats(|info| { - assert_eq!(3, info.stats.compile_requests); - assert_eq!(15, info.stats.requests_executed); - assert_eq!(5, info.stats.cache_hits.all()); - assert_eq!(7, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA").unwrap()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA (Device code)").unwrap()); - assert_eq!(&1, info.stats.cache_hits.get("PTX").unwrap()); - assert_eq!(&2, info.stats.cache_hits.get("CUBIN").unwrap()); - assert_eq!(&2, info.stats.cache_misses.get("CUDA").unwrap()); - assert_eq!( - &2, - info.stats.cache_misses.get("CUDA (Device code)").unwrap() - ); - assert_eq!(&2, info.stats.cache_misses.get("PTX").unwrap()); - assert_eq!(&1, info.stats.cache_misses.get("CUBIN").unwrap()); - assert!(info.stats.cache_misses.get("C/C++").is_none()); - let adv_cuda_key = adv_key_kind("cuda", compiler.name); - let adv_cudafe_key = adv_key_kind("cudafe++", compiler.name); - let adv_ptx_key = adv_key_kind("ptx", compiler.name); - let adv_cubin_key = adv_key_kind("cubin", compiler.name); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_cuda_key).unwrap()); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_cudafe_key).unwrap()); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_ptx_key).unwrap()); - assert_eq!(&2, info.stats.cache_hits.get_adv(&adv_cubin_key).unwrap()); - assert_eq!(&2, info.stats.cache_misses.get_adv(&adv_cuda_key).unwrap()); - assert_eq!( - &2, - info.stats.cache_misses.get_adv(&adv_cudafe_key).unwrap() - ); - assert_eq!(&2, info.stats.cache_misses.get_adv(&adv_ptx_key).unwrap()); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_cubin_key).unwrap()); - }); + run_cuda_test( + "-c", + &tempdir.join(INPUT_FOR_CUDA_B), // absolute path for input + &tempdir.join(&build_dir).join(OUTPUT), // absolute path for output + &extra_args, + AdditionalStats { + cache_writes: Some(3), + compilations: Some(4), + compile_requests: Some(1), + requests_executed: Some(5), + cache_hits: Some(vec![(CCompilerKind::Ptxas, Language::Cubin, 1)]), + cache_misses: Some(vec![ + (CCompilerKind::CudaFE, Language::CudaFE, 1), + (CCompilerKind::Cicc, Language::Ptx, 1), + (CCompilerKind::Nvcc, Language::Cuda, 1), + ]), + ..Default::default() + }, + ); trace!("compile ptx"); - let out_file = tempdir.join("test.ptx"); - sccache_command() - .args(compile_cuda_cmdline( - name, - exe, - "-ptx", - INPUT_FOR_CUDA_A, - // relative path for output - out_file.file_name().unwrap().to_string_lossy().as_ref(), - Vec::new(), - )) - .current_dir(tempdir) - .envs(env_vars.clone()) - .assert() - .success(); - assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); - fs::remove_file(&out_file).unwrap(); - trace!("compile ptx request stats"); - get_stats(|info| { - assert_eq!(4, info.stats.compile_requests); - assert_eq!(17, info.stats.requests_executed); - assert_eq!(5, info.stats.cache_hits.all()); - assert_eq!(8, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA").unwrap()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA (Device code)").unwrap()); - assert_eq!(&1, info.stats.cache_hits.get("PTX").unwrap()); - assert_eq!(&2, info.stats.cache_hits.get("CUBIN").unwrap()); - assert_eq!(&2, info.stats.cache_misses.get("CUDA").unwrap()); - assert_eq!( - &2, - info.stats.cache_misses.get("CUDA (Device code)").unwrap() - ); - assert_eq!(&3, info.stats.cache_misses.get("PTX").unwrap()); - assert_eq!(&1, info.stats.cache_misses.get("CUBIN").unwrap()); - assert!(info.stats.cache_misses.get("C/C++").is_none()); - let adv_cuda_key = adv_key_kind("cuda", compiler.name); - let adv_cudafe_key = adv_key_kind("cudafe++", compiler.name); - let adv_ptx_key = adv_key_kind("ptx", compiler.name); - let adv_cubin_key = adv_key_kind("cubin", compiler.name); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_cuda_key).unwrap()); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_cudafe_key).unwrap()); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_ptx_key).unwrap()); - assert_eq!(&2, info.stats.cache_hits.get_adv(&adv_cubin_key).unwrap()); - assert_eq!(&2, info.stats.cache_misses.get_adv(&adv_cuda_key).unwrap()); - assert_eq!( - &2, - info.stats.cache_misses.get_adv(&adv_cudafe_key).unwrap() - ); - assert_eq!(&3, info.stats.cache_misses.get_adv(&adv_ptx_key).unwrap()); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_cubin_key).unwrap()); - }); + run_cuda_test( + "-ptx", + Path::new(INPUT_FOR_CUDA_A), // relative path for input + &build_dir.join("test.ptx"), // relative path for output + &extra_args, + AdditionalStats { + cache_writes: Some(1), + compilations: Some(2), + compile_requests: Some(1), + requests_executed: Some(2), + cache_misses: Some(vec![(CCompilerKind::Cicc, Language::Ptx, 1)]), + ..Default::default() + }, + ); trace!("compile cubin"); - let out_file = tempdir.join("test.cubin"); - sccache_command() - .args(compile_cuda_cmdline( - name, - exe, - "-cubin", - INPUT_FOR_CUDA_A, - // absolute path for output - out_file.to_string_lossy().as_ref(), - Vec::new(), - )) - .current_dir(tempdir) - .envs(env_vars.clone()) - .assert() - .success(); - assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); - fs::remove_file(&out_file).unwrap(); - trace!("compile cubin request stats"); - get_stats(|info| { - assert_eq!(5, info.stats.compile_requests); - assert_eq!(20, info.stats.requests_executed); - assert_eq!(6, info.stats.cache_hits.all()); - assert_eq!(9, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA").unwrap()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA (Device code)").unwrap()); - assert_eq!(&1, info.stats.cache_hits.get("PTX").unwrap()); - assert_eq!(&3, info.stats.cache_hits.get("CUBIN").unwrap()); - assert_eq!(&2, info.stats.cache_misses.get("CUDA").unwrap()); - assert_eq!( - &2, - info.stats.cache_misses.get("CUDA (Device code)").unwrap() - ); - assert_eq!(&4, info.stats.cache_misses.get("PTX").unwrap()); - assert_eq!(&1, info.stats.cache_misses.get("CUBIN").unwrap()); - assert!(info.stats.cache_misses.get("C/C++").is_none()); - let adv_cuda_key = adv_key_kind("cuda", compiler.name); - let adv_cudafe_key = adv_key_kind("cudafe++", compiler.name); - let adv_ptx_key = adv_key_kind("ptx", compiler.name); - let adv_cubin_key = adv_key_kind("cubin", compiler.name); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_cuda_key).unwrap()); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_cudafe_key).unwrap()); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_ptx_key).unwrap()); - assert_eq!(&3, info.stats.cache_hits.get_adv(&adv_cubin_key).unwrap()); - assert_eq!(&2, info.stats.cache_misses.get_adv(&adv_cuda_key).unwrap()); - assert_eq!( - &2, - info.stats.cache_misses.get_adv(&adv_cudafe_key).unwrap() - ); - assert_eq!(&4, info.stats.cache_misses.get_adv(&adv_ptx_key).unwrap()); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_cubin_key).unwrap()); - }); + run_cuda_test( + "-cubin", + Path::new(INPUT_FOR_CUDA_A), // relative path for input + &tempdir.join(&build_dir).join("test.cubin"), // absolute path for output + &extra_args, + AdditionalStats { + cache_writes: Some(1), + compilations: Some(2), + compile_requests: Some(1), + requests_executed: Some(3), + cache_hits: Some(vec![ + // TODO: Fix this in the next PR + // (CCompilerKind::Cicc, Language::Ptx, 1), + (CCompilerKind::Ptxas, Language::Cubin, 1), + ]), + // TODO: Should not be a cache miss. + // Fix this in the next PR + cache_misses: Some(vec![(CCompilerKind::Cicc, Language::Ptx, 1)]), + ..Default::default() + }, + ); // Test to ensure #2299 doesn't regress (https://github.com/mozilla/sccache/issues/2299) let test_2299_src_name = "test_2299.cu"; - let test_2299_out_file = tempdir.join("test_2299.cu.o"); + let test_2299_out_name = "test_2299.cu.o"; // Two versions of the source with different contents inside the #ifndef __CUDA_ARCH__ let test_2299_cu_src_1 = " #ifndef __CUDA_ARCH__ @@ -979,204 +934,100 @@ int main(int argc, char** argv) { "; write_source(tempdir, test_2299_src_name, test_2299_cu_src_1); trace!("compile test_2299.cu (1)"); - sccache_command() - .args(compile_cuda_cmdline( - name, - exe, - "-c", - // relative path for input - test_2299_src_name, - // relative path for output - test_2299_out_file - .file_name() - .unwrap() - .to_string_lossy() - .as_ref(), - Vec::new(), - )) - .current_dir(tempdir) - .envs(env_vars.clone()) - .assert() - .success(); - assert!(fs::metadata(&test_2299_out_file) - .map(|m| m.len() > 0) - .unwrap()); - fs::remove_file(&test_2299_out_file).unwrap(); - trace!("compile test_2299.cu request stats (1)"); - get_stats(|info| { - assert_eq!(6, info.stats.compile_requests); - assert_eq!(25, info.stats.requests_executed); - assert_eq!(6, info.stats.cache_hits.all()); - assert_eq!(13, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA").unwrap()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA (Device code)").unwrap()); - assert_eq!(&1, info.stats.cache_hits.get("PTX").unwrap()); - assert_eq!(&3, info.stats.cache_hits.get("CUBIN").unwrap()); - assert_eq!(&3, info.stats.cache_misses.get("CUDA").unwrap()); - assert_eq!( - &3, - info.stats.cache_misses.get("CUDA (Device code)").unwrap() - ); - assert_eq!(&5, info.stats.cache_misses.get("PTX").unwrap()); - assert_eq!(&2, info.stats.cache_misses.get("CUBIN").unwrap()); - assert!(info.stats.cache_misses.get("C/C++").is_none()); - let adv_cuda_key = adv_key_kind("cuda", compiler.name); - let adv_cudafe_key = adv_key_kind("cudafe++", compiler.name); - let adv_ptx_key = adv_key_kind("ptx", compiler.name); - let adv_cubin_key = adv_key_kind("cubin", compiler.name); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_cuda_key).unwrap()); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_cudafe_key).unwrap()); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_ptx_key).unwrap()); - assert_eq!(&3, info.stats.cache_hits.get_adv(&adv_cubin_key).unwrap()); - assert_eq!(&3, info.stats.cache_misses.get_adv(&adv_cuda_key).unwrap()); - assert_eq!( - &3, - info.stats.cache_misses.get_adv(&adv_cudafe_key).unwrap() - ); - assert_eq!(&5, info.stats.cache_misses.get_adv(&adv_ptx_key).unwrap()); - assert_eq!(&2, info.stats.cache_misses.get_adv(&adv_cubin_key).unwrap()); - }); + run_cuda_test( + "-c", + Path::new(test_2299_src_name), // relative path for input + &build_dir.join(test_2299_out_name), // relative path for output + &extra_args, + AdditionalStats { + cache_writes: Some(4), + compilations: Some(5), + compile_requests: Some(1), + requests_executed: Some(5), + cache_misses: Some(vec![ + (CCompilerKind::Nvcc, Language::Cuda, 1), + (CCompilerKind::CudaFE, Language::CudaFE, 1), + (CCompilerKind::Cicc, Language::Ptx, 1), + (CCompilerKind::Ptxas, Language::Cubin, 1), + ]), + ..Default::default() + }, + ); write_source(tempdir, test_2299_src_name, test_2299_cu_src_2); trace!("compile test_2299.cu (2)"); - sccache_command() - .args(compile_cuda_cmdline( - name, - exe, - "-c", - // relative path for input - test_2299_src_name, - // relative path for output - test_2299_out_file - .file_name() - .unwrap() - .to_string_lossy() - .as_ref(), - Vec::new(), - )) - .current_dir(tempdir) - .envs(env_vars.clone()) - .assert() - .success(); - assert!(fs::metadata(&test_2299_out_file) - .map(|m| m.len() > 0) - .unwrap()); - fs::remove_file(&test_2299_out_file).unwrap(); - trace!("compile test_2299.cu request stats (2)"); - get_stats(|info| { - assert_eq!(7, info.stats.compile_requests); - assert_eq!(30, info.stats.requests_executed); - assert_eq!(8, info.stats.cache_hits.all()); - assert_eq!(15, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA").unwrap()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA (Device code)").unwrap()); - assert_eq!(&2, info.stats.cache_hits.get("PTX").unwrap()); - assert_eq!(&4, info.stats.cache_hits.get("CUBIN").unwrap()); - assert_eq!(&4, info.stats.cache_misses.get("CUDA").unwrap()); - assert_eq!( - &4, - info.stats.cache_misses.get("CUDA (Device code)").unwrap() - ); - assert_eq!(&5, info.stats.cache_misses.get("PTX").unwrap()); - assert_eq!(&2, info.stats.cache_misses.get("CUBIN").unwrap()); - assert!(info.stats.cache_misses.get("C/C++").is_none()); - let adv_cuda_key = adv_key_kind("cuda", compiler.name); - let adv_cudafe_key = adv_key_kind("cudafe++", compiler.name); - let adv_ptx_key = adv_key_kind("ptx", compiler.name); - let adv_cubin_key = adv_key_kind("cubin", compiler.name); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_cuda_key).unwrap()); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_cudafe_key).unwrap()); - assert_eq!(&2, info.stats.cache_hits.get_adv(&adv_ptx_key).unwrap()); - assert_eq!(&4, info.stats.cache_hits.get_adv(&adv_cubin_key).unwrap()); - assert_eq!(&4, info.stats.cache_misses.get_adv(&adv_cuda_key).unwrap()); - assert_eq!( - &4, - info.stats.cache_misses.get_adv(&adv_cudafe_key).unwrap() - ); - assert_eq!(&5, info.stats.cache_misses.get_adv(&adv_ptx_key).unwrap()); - assert_eq!(&2, info.stats.cache_misses.get_adv(&adv_cubin_key).unwrap()); - }); + run_cuda_test( + "-c", + Path::new(test_2299_src_name), // relative path for input + &build_dir.join(test_2299_out_name), // relative path for output + &extra_args, + AdditionalStats { + cache_writes: Some(2), + compilations: Some(3), + compile_requests: Some(1), + requests_executed: Some(5), + cache_hits: Some(vec![ + (CCompilerKind::Cicc, Language::Ptx, 1), + (CCompilerKind::Ptxas, Language::Cubin, 1), + ]), + cache_misses: Some(vec![ + (CCompilerKind::Nvcc, Language::Cuda, 1), + (CCompilerKind::CudaFE, Language::CudaFE, 1), + ]), + ..Default::default() + }, + ); // Recompile the original version again to ensure only cache hits write_source(tempdir, test_2299_src_name, test_2299_cu_src_1); trace!("compile test_2299.cu (3)"); - sccache_command() - .args(compile_cuda_cmdline( - name, - exe, - "-c", - // relative path for input - test_2299_src_name, - // relative path for output - test_2299_out_file - .file_name() - .unwrap() - .to_string_lossy() - .as_ref(), - Vec::new(), - )) - .current_dir(tempdir) - .envs(env_vars.clone()) - .assert() - .success(); - assert!(fs::metadata(&test_2299_out_file) - .map(|m| m.len() > 0) - .unwrap()); - fs::remove_file(&test_2299_out_file).unwrap(); - trace!("compile test_2299.cu request stats (3)"); - get_stats(|info| { - assert_eq!(8, info.stats.compile_requests); - assert_eq!(35, info.stats.requests_executed); - assert_eq!(12, info.stats.cache_hits.all()); - assert_eq!(15, info.stats.cache_misses.all()); - assert_eq!(&2, info.stats.cache_hits.get("CUDA").unwrap()); - assert_eq!(&2, info.stats.cache_hits.get("CUDA (Device code)").unwrap()); - assert_eq!(&3, info.stats.cache_hits.get("PTX").unwrap()); - assert_eq!(&5, info.stats.cache_hits.get("CUBIN").unwrap()); - assert_eq!(&4, info.stats.cache_misses.get("CUDA").unwrap()); - assert_eq!( - &4, - info.stats.cache_misses.get("CUDA (Device code)").unwrap() - ); - assert_eq!(&5, info.stats.cache_misses.get("PTX").unwrap()); - assert_eq!(&2, info.stats.cache_misses.get("CUBIN").unwrap()); - assert!(info.stats.cache_misses.get("C/C++").is_none()); - let adv_cuda_key = adv_key_kind("cuda", compiler.name); - let adv_cudafe_key = adv_key_kind("cudafe++", compiler.name); - let adv_ptx_key = adv_key_kind("ptx", compiler.name); - let adv_cubin_key = adv_key_kind("cubin", compiler.name); - assert_eq!(&2, info.stats.cache_hits.get_adv(&adv_cuda_key).unwrap()); - assert_eq!(&2, info.stats.cache_hits.get_adv(&adv_cudafe_key).unwrap()); - assert_eq!(&3, info.stats.cache_hits.get_adv(&adv_ptx_key).unwrap()); - assert_eq!(&5, info.stats.cache_hits.get_adv(&adv_cubin_key).unwrap()); - assert_eq!(&4, info.stats.cache_misses.get_adv(&adv_cuda_key).unwrap()); - assert_eq!( - &4, - info.stats.cache_misses.get_adv(&adv_cudafe_key).unwrap() - ); - assert_eq!(&5, info.stats.cache_misses.get_adv(&adv_ptx_key).unwrap()); - assert_eq!(&2, info.stats.cache_misses.get_adv(&adv_cubin_key).unwrap()); - }); + run_cuda_test( + "-c", + &tempdir.join(test_2299_src_name), // absolute path for input + &tempdir.join(&build_dir).join(test_2299_out_name), // absolute path for output + &extra_args, + AdditionalStats { + compilations: Some(1), + compile_requests: Some(1), + requests_executed: Some(5), + cache_hits: Some(vec![ + (CCompilerKind::Nvcc, Language::Cuda, 1), + (CCompilerKind::CudaFE, Language::CudaFE, 1), + (CCompilerKind::Cicc, Language::Ptx, 1), + (CCompilerKind::Ptxas, Language::Cubin, 1), + ]), + ..Default::default() + }, + ); } -fn test_nvcc_proper_lang_stat_tracking(compiler: Compiler, tempdir: &Path) { +fn test_nvcc_proper_lang_stat_tracking( + client: &SccacheClient, + compiler: &Compiler, + tempdir: &Path, +) { + let mut stats = client.stats().unwrap(); + + let extra_args = vec![]; + let Compiler { name, exe, env_vars, } = compiler; - zero_stats(); println!("test_nvcc_proper_lang_stat_tracking: {}", name); // Compile multiple source files. copy_to_tempdir(&[INPUT_FOR_CUDA_C, INPUT], tempdir); let out_file = tempdir.join(OUTPUT); - trace!("compile CUDA A"); - sccache_command() + + trace!("compile CUDA C"); + client + .cmd() .args(compile_cmdline( name, - &exe, + exe, INPUT_FOR_CUDA_C, OUTPUT, Vec::new(), @@ -1186,63 +1037,154 @@ fn test_nvcc_proper_lang_stat_tracking(compiler: Compiler, tempdir: &Path) { .assert() .success(); fs::remove_file(&out_file).unwrap(); - trace!("compile CUDA A"); - sccache_command() + + stats.cache_writes += 3; + stats.compilations += 4; + stats.compile_requests += 1; + stats.requests_executed += 5; + stats.non_cacheable_compilations += 1; + stats + .cache_misses + .increment(&CompilerKind::C(CCompilerKind::Nvcc), &Language::Cuda); + stats + .cache_misses + .increment(&CompilerKind::C(CCompilerKind::CudaFE), &Language::CudaFE); + stats + .cache_misses + .increment(&CompilerKind::C(CCompilerKind::Cicc), &Language::Ptx); + stats + .cache_hits + .increment(&CompilerKind::C(CCompilerKind::Ptxas), &Language::Cubin); + assert_eq!( + stats, + ServerStats { + cache_write_duration: stats.cache_write_duration, + cache_read_hit_duration: stats.cache_read_hit_duration, + compiler_write_duration: stats.compiler_write_duration, + ..client.stats().unwrap() + } + ); + + trace!("compile CUDA C"); + client + .cmd() .args(compile_cmdline( name, - &exe, + exe, INPUT_FOR_CUDA_C, OUTPUT, - Vec::new(), + extra_args.clone(), )) .current_dir(tempdir) .envs(env_vars.clone()) .assert() .success(); fs::remove_file(&out_file).unwrap(); - trace!("compile C++ A"); - sccache_command() - .args(compile_cmdline(name, &exe, INPUT, OUTPUT, Vec::new())) + + stats.compilations += 1; + stats.compile_requests += 1; + stats.requests_executed += 5; + stats.non_cacheable_compilations += 1; + stats + .cache_hits + .increment(&CompilerKind::C(CCompilerKind::Nvcc), &Language::Cuda); + stats + .cache_hits + .increment(&CompilerKind::C(CCompilerKind::CudaFE), &Language::CudaFE); + stats + .cache_hits + .increment(&CompilerKind::C(CCompilerKind::Cicc), &Language::Ptx); + stats + .cache_hits + .increment(&CompilerKind::C(CCompilerKind::Ptxas), &Language::Cubin); + assert_eq!( + stats, + ServerStats { + cache_write_duration: stats.cache_write_duration, + cache_read_hit_duration: stats.cache_read_hit_duration, + compiler_write_duration: stats.compiler_write_duration, + ..client.stats().unwrap() + } + ); + + trace!("compile C++"); + client + .cmd() + .args(compile_cmdline( + name, + exe, + INPUT, + OUTPUT, + extra_args.clone(), + )) .current_dir(tempdir) .envs(env_vars.clone()) .assert() .success(); fs::remove_file(&out_file).unwrap(); - trace!("compile C++ A"); - sccache_command() - .args(compile_cmdline(name, &exe, INPUT, OUTPUT, Vec::new())) + + stats.cache_writes += 1; + stats.compilations += 2; + stats.compile_requests += 1; + stats.requests_executed += 2; + stats.non_cacheable_compilations += 1; + stats + .cache_misses + .increment(&CompilerKind::C(CCompilerKind::Nvcc), &Language::Cuda); + assert_eq!( + stats, + ServerStats { + cache_write_duration: stats.cache_write_duration, + cache_read_hit_duration: stats.cache_read_hit_duration, + compiler_write_duration: stats.compiler_write_duration, + ..client.stats().unwrap() + } + ); + + trace!("compile C++"); + client + .cmd() + .args(compile_cmdline( + name, + exe, + INPUT, + OUTPUT, + extra_args.clone(), + )) .current_dir(tempdir) - .envs(env_vars) + .envs(env_vars.clone()) .assert() .success(); fs::remove_file(&out_file).unwrap(); - trace!("request stats"); - get_stats(|info| { - assert_eq!(4, info.stats.compile_requests); - assert_eq!(14, info.stats.requests_executed); - assert_eq!(6, info.stats.cache_hits.all()); - assert_eq!(4, info.stats.cache_misses.all()); - assert!(info.stats.cache_hits.get("C/C++").is_none()); - assert_eq!(&2, info.stats.cache_hits.get("CUDA").unwrap()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA (Device code)").unwrap()); - assert_eq!(&2, info.stats.cache_hits.get("CUBIN").unwrap()); - assert!(info.stats.cache_misses.get("C/C++").is_none()); - assert_eq!(&2, info.stats.cache_misses.get("CUDA").unwrap()); - assert_eq!( - &1, - info.stats.cache_misses.get("CUDA (Device code)").unwrap() - ); - assert_eq!(&1, info.stats.cache_misses.get("PTX").unwrap()); - }); + stats.compilations += 1; + stats.compile_requests += 1; + stats.requests_executed += 2; + stats.non_cacheable_compilations += 1; + stats + .cache_hits + .increment(&CompilerKind::C(CCompilerKind::Nvcc), &Language::Cuda); + assert_eq!( + stats, + ServerStats { + cache_write_duration: stats.cache_write_duration, + cache_read_hit_duration: stats.cache_read_hit_duration, + compiler_write_duration: stats.compiler_write_duration, + ..client.stats().unwrap() + } + ); } -fn run_sccache_nvcc_cuda_command_tests(compiler: Compiler, tempdir: &Path) { - test_nvcc_cuda_compiles(&compiler, tempdir); - test_nvcc_proper_lang_stat_tracking(compiler, tempdir); +fn run_sccache_nvcc_cuda_command_tests(client: &SccacheClient, compiler: Compiler, tempdir: &Path) { + test_nvcc_cuda_compiles(client, &compiler, tempdir); + test_nvcc_proper_lang_stat_tracking(client, &compiler, tempdir); } -fn test_clang_cuda_compiles(compiler: &Compiler, tempdir: &Path) { +fn test_clang_cuda_compiles(client: &SccacheClient, compiler: &Compiler, tempdir: &Path) { + let mut stats = client.stats().unwrap(); + + let extra_args = vec![]; + let Compiler { name, exe, @@ -1254,96 +1196,120 @@ fn test_clang_cuda_compiles(compiler: &Compiler, tempdir: &Path) { let out_file = tempdir.join(OUTPUT); trace!("compile A"); - sccache_command() + client + .cmd() .args(compile_cuda_cmdline( name, exe, "-c", INPUT_FOR_CUDA_A, OUTPUT, - Vec::new(), + &extra_args, )) .current_dir(tempdir) .envs(env_vars.clone()) .assert() .success(); assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); - trace!("request stats"); - get_stats(|info| { - assert_eq!(1, info.stats.compile_requests); - assert_eq!(1, info.stats.requests_executed); - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_misses.get("CUDA").unwrap()); - let adv_cuda_key = adv_key_kind("cuda", compiler.name); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_cuda_key).unwrap()); - }); - trace!("compile A"); fs::remove_file(&out_file).unwrap(); - sccache_command() + stats.cache_writes += 1; + stats.compilations += 1; + stats.compile_requests += 1; + stats.requests_executed += 1; + stats + .cache_misses + .increment(&CompilerKind::C(CCompilerKind::Clang), &Language::Cuda); + assert_eq!( + stats, + ServerStats { + cache_write_duration: stats.cache_write_duration, + cache_read_hit_duration: stats.cache_read_hit_duration, + compiler_write_duration: stats.compiler_write_duration, + ..client.stats().unwrap() + } + ); + + trace!("compile A"); + client + .cmd() .args(compile_cuda_cmdline( name, exe, "-c", INPUT_FOR_CUDA_A, OUTPUT, - Vec::new(), + &extra_args, )) .current_dir(tempdir) .envs(env_vars.clone()) .assert() .success(); assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); - trace!("request stats"); - get_stats(|info| { - assert_eq!(2, info.stats.compile_requests); - assert_eq!(2, info.stats.requests_executed); - assert_eq!(1, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA").unwrap()); - assert_eq!(&1, info.stats.cache_misses.get("CUDA").unwrap()); - let adv_cuda_key = adv_key_kind("cuda", compiler.name); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_cuda_key).unwrap()); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_cuda_key).unwrap()); - }); + fs::remove_file(&out_file).unwrap(); + stats.compile_requests += 1; + stats.requests_executed += 1; + stats + .cache_hits + .increment(&CompilerKind::C(CCompilerKind::Clang), &Language::Cuda); + assert_eq!( + stats, + ServerStats { + cache_write_duration: stats.cache_write_duration, + cache_read_hit_duration: stats.cache_read_hit_duration, + compiler_write_duration: stats.compiler_write_duration, + ..client.stats().unwrap() + } + ); + // By compiling another input source we verify that the pre-processor // phase is correctly running and outputting text trace!("compile B"); - sccache_command() + client + .cmd() .args(compile_cuda_cmdline( name, exe, "-c", INPUT_FOR_CUDA_B, OUTPUT, - Vec::new(), + &extra_args, )) .current_dir(tempdir) .envs(env_vars.clone()) .assert() .success(); assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); - trace!("request stats"); - get_stats(|info| { - assert_eq!(3, info.stats.compile_requests); - assert_eq!(3, info.stats.requests_executed); - assert_eq!(1, info.stats.cache_hits.all()); - assert_eq!(2, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA").unwrap()); - assert_eq!(&2, info.stats.cache_misses.get("CUDA").unwrap()); - let adv_cuda_key = adv_key_kind("cuda", compiler.name); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_cuda_key).unwrap()); - assert_eq!(&2, info.stats.cache_misses.get_adv(&adv_cuda_key).unwrap()); - }); + fs::remove_file(&out_file).unwrap(); + stats.cache_writes += 1; + stats.compilations += 1; + stats.compile_requests += 1; + stats.requests_executed += 1; + stats + .cache_misses + .increment(&CompilerKind::C(CCompilerKind::Clang), &Language::Cuda); + assert_eq!( + stats, + ServerStats { + cache_write_duration: stats.cache_write_duration, + cache_read_hit_duration: stats.cache_read_hit_duration, + compiler_write_duration: stats.compiler_write_duration, + ..client.stats().unwrap() + } + ); } -fn test_clang_proper_lang_stat_tracking(compiler: Compiler, tempdir: &Path) { +fn test_clang_proper_lang_stat_tracking( + client: &SccacheClient, + compiler: &Compiler, + tempdir: &Path, +) { + let mut stats = client.stats().unwrap(); + let Compiler { name, exe, env_vars, } = compiler; - zero_stats(); println!("test_clang_proper_lang_stat_tracking: {}", name); // Compile multiple source files. @@ -1351,71 +1317,130 @@ fn test_clang_proper_lang_stat_tracking(compiler: Compiler, tempdir: &Path) { let out_file = tempdir.join(OUTPUT); trace!("compile CUDA A"); - sccache_command() + client + .cmd() .args(compile_cuda_cmdline( name, - &exe, + exe, "-c", INPUT_FOR_CUDA_C, OUTPUT, - Vec::new(), + &[], )) .current_dir(tempdir) .envs(env_vars.clone()) .assert() .success(); fs::remove_file(&out_file).unwrap(); + stats.cache_writes += 1; + stats.compilations += 1; + stats.compile_requests += 1; + stats.requests_executed += 1; + stats + .cache_misses + .increment(&CompilerKind::C(CCompilerKind::Clang), &Language::Cuda); + assert_eq!( + stats, + ServerStats { + cache_write_duration: stats.cache_write_duration, + cache_read_hit_duration: stats.cache_read_hit_duration, + compiler_write_duration: stats.compiler_write_duration, + ..client.stats().unwrap() + } + ); + trace!("compile CUDA A"); - sccache_command() + client + .cmd() .args(compile_cuda_cmdline( name, - &exe, + exe, "-c", INPUT_FOR_CUDA_C, OUTPUT, - Vec::new(), + &[], )) .current_dir(tempdir) .envs(env_vars.clone()) .assert() .success(); fs::remove_file(&out_file).unwrap(); + stats.compile_requests += 1; + stats.requests_executed += 1; + stats + .cache_hits + .increment(&CompilerKind::C(CCompilerKind::Clang), &Language::Cuda); + assert_eq!( + stats, + ServerStats { + cache_write_duration: stats.cache_write_duration, + cache_read_hit_duration: stats.cache_read_hit_duration, + compiler_write_duration: stats.compiler_write_duration, + ..client.stats().unwrap() + } + ); + trace!("compile C++ A"); - sccache_command() - .args(compile_cmdline(name, &exe, INPUT, OUTPUT, Vec::new())) + client + .cmd() + .args(compile_cmdline(name, exe, INPUT, OUTPUT, Vec::new())) .current_dir(tempdir) .envs(env_vars.clone()) .assert() .success(); fs::remove_file(&out_file).unwrap(); + stats.cache_writes += 1; + stats.compilations += 1; + stats.compile_requests += 1; + stats.requests_executed += 1; + stats + .cache_misses + .increment(&CompilerKind::C(CCompilerKind::Clang), &Language::Cxx); + assert_eq!( + stats, + ServerStats { + cache_write_duration: stats.cache_write_duration, + cache_read_hit_duration: stats.cache_read_hit_duration, + compiler_write_duration: stats.compiler_write_duration, + ..client.stats().unwrap() + } + ); + trace!("compile C++ A"); - sccache_command() - .args(compile_cmdline(name, &exe, INPUT, OUTPUT, Vec::new())) + client + .cmd() + .args(compile_cmdline(name, exe, INPUT, OUTPUT, Vec::new())) .current_dir(tempdir) - .envs(env_vars) + .envs(env_vars.clone()) .assert() .success(); fs::remove_file(&out_file).unwrap(); - - trace!("request stats"); - get_stats(|info| { - assert_eq!(4, info.stats.compile_requests); - assert_eq!(4, info.stats.requests_executed); - assert_eq!(2, info.stats.cache_hits.all()); - assert_eq!(2, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("C/C++").unwrap()); - assert_eq!(&1, info.stats.cache_misses.get("C/C++").unwrap()); - assert_eq!(&1, info.stats.cache_hits.get("CUDA").unwrap()); - assert_eq!(&1, info.stats.cache_misses.get("CUDA").unwrap()); - }); + stats.compile_requests += 1; + stats.requests_executed += 1; + stats + .cache_hits + .increment(&CompilerKind::C(CCompilerKind::Clang), &Language::Cxx); + assert_eq!( + stats, + ServerStats { + cache_write_duration: stats.cache_write_duration, + cache_read_hit_duration: stats.cache_read_hit_duration, + compiler_write_duration: stats.compiler_write_duration, + ..client.stats().unwrap() + } + ); } -fn run_sccache_clang_cuda_command_tests(compiler: Compiler, tempdir: &Path) { - test_clang_cuda_compiles(&compiler, tempdir); - test_clang_proper_lang_stat_tracking(compiler, tempdir); +fn run_sccache_clang_cuda_command_tests( + client: &SccacheClient, + compiler: Compiler, + tempdir: &Path, +) { + test_clang_cuda_compiles(client, &compiler, tempdir); + test_clang_proper_lang_stat_tracking(client, &compiler, tempdir); } -fn test_hip_compiles(compiler: &Compiler, tempdir: &Path) { +fn test_hip_compiles(client: &SccacheClient, compiler: &Compiler, tempdir: &Path) { let Compiler { name, exe, @@ -1429,7 +1454,8 @@ fn test_hip_compiles(compiler: &Compiler, tempdir: &Path) { let out_file = tempdir.join(OUTPUT); trace!("compile A"); - sccache_command() + client + .cmd() .args(compile_hip_cmdline( name, exe, @@ -1444,18 +1470,18 @@ fn test_hip_compiles(compiler: &Compiler, tempdir: &Path) { .success(); assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); trace!("request stats"); - get_stats(|info| { - assert_eq!(1, info.stats.compile_requests); - assert_eq!(1, info.stats.requests_executed); - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_misses.get("HIP").unwrap()); - let adv_hip_key = adv_key_kind("hip", compiler.name); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_hip_key).unwrap()); - }); + let stats = client.stats().unwrap(); + assert_eq!(1, stats.compile_requests); + assert_eq!(1, stats.requests_executed); + assert_eq!(0, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); + assert_eq!(&1, stats.cache_misses.get("HIP").unwrap()); + let adv_hip_key = adv_key_kind("hip", compiler.name); + assert_eq!(&1, stats.cache_misses.get_adv(&adv_hip_key).unwrap()); trace!("compile A"); fs::remove_file(&out_file).unwrap(); - sccache_command() + client + .cmd() .args(compile_hip_cmdline( name, exe, @@ -1470,21 +1496,21 @@ fn test_hip_compiles(compiler: &Compiler, tempdir: &Path) { .success(); assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); trace!("request stats"); - get_stats(|info| { - assert_eq!(2, info.stats.compile_requests); - assert_eq!(2, info.stats.requests_executed); - assert_eq!(1, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("HIP").unwrap()); - assert_eq!(&1, info.stats.cache_misses.get("HIP").unwrap()); - let adv_hip_key = adv_key_kind("hip", compiler.name); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_hip_key).unwrap()); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_hip_key).unwrap()); - }); + let stats = client.stats().unwrap(); + assert_eq!(2, stats.compile_requests); + assert_eq!(2, stats.requests_executed); + assert_eq!(1, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); + assert_eq!(&1, stats.cache_hits.get("HIP").unwrap()); + assert_eq!(&1, stats.cache_misses.get("HIP").unwrap()); + let adv_hip_key = adv_key_kind("hip", compiler.name); + assert_eq!(&1, stats.cache_hits.get_adv(&adv_hip_key).unwrap()); + assert_eq!(&1, stats.cache_misses.get_adv(&adv_hip_key).unwrap()); // By compiling another input source we verify that the pre-processor // phase is correctly running and outputting text trace!("compile B"); - sccache_command() + client + .cmd() .args(compile_hip_cmdline( name, exe, @@ -1499,20 +1525,19 @@ fn test_hip_compiles(compiler: &Compiler, tempdir: &Path) { .success(); assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); trace!("request stats"); - get_stats(|info| { - assert_eq!(3, info.stats.compile_requests); - assert_eq!(3, info.stats.requests_executed); - assert_eq!(1, info.stats.cache_hits.all()); - assert_eq!(2, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("HIP").unwrap()); - assert_eq!(&2, info.stats.cache_misses.get("HIP").unwrap()); - let adv_hip_key = adv_key_kind("hip", compiler.name); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_hip_key).unwrap()); - assert_eq!(&2, info.stats.cache_misses.get_adv(&adv_hip_key).unwrap()); - }); + let stats = client.stats().unwrap(); + assert_eq!(3, stats.compile_requests); + assert_eq!(3, stats.requests_executed); + assert_eq!(1, stats.cache_hits.all()); + assert_eq!(2, stats.cache_misses.all()); + assert_eq!(&1, stats.cache_hits.get("HIP").unwrap()); + assert_eq!(&2, stats.cache_misses.get("HIP").unwrap()); + let adv_hip_key = adv_key_kind("hip", compiler.name); + assert_eq!(&1, stats.cache_hits.get_adv(&adv_hip_key).unwrap()); + assert_eq!(&2, stats.cache_misses.get_adv(&adv_hip_key).unwrap()); } -fn test_hip_compiles_multi_targets(compiler: &Compiler, tempdir: &Path) { +fn test_hip_compiles_multi_targets(client: &SccacheClient, compiler: &Compiler, tempdir: &Path) { let Compiler { name, exe, @@ -1526,7 +1551,8 @@ fn test_hip_compiles_multi_targets(compiler: &Compiler, tempdir: &Path) { let out_file = tempdir.join(OUTPUT); trace!("compile A with gfx900 and gfx1030"); - sccache_command() + client + .cmd() .args(compile_hip_cmdline( name, exe, @@ -1541,19 +1567,19 @@ fn test_hip_compiles_multi_targets(compiler: &Compiler, tempdir: &Path) { .success(); assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); trace!("request stats"); - get_stats(|info| { - assert_eq!(1, info.stats.compile_requests); - assert_eq!(1, info.stats.requests_executed); - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_misses.get("HIP").unwrap()); - let adv_hip_key = adv_key_kind("hip", compiler.name); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_hip_key).unwrap()); - }); + let stats = client.stats().unwrap(); + assert_eq!(1, stats.compile_requests); + assert_eq!(1, stats.requests_executed); + assert_eq!(0, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); + assert_eq!(&1, stats.cache_misses.get("HIP").unwrap()); + let adv_hip_key = adv_key_kind("hip", compiler.name); + assert_eq!(&1, stats.cache_misses.get_adv(&adv_hip_key).unwrap()); trace!("compile A with with gfx900 and gfx1030 again"); fs::remove_file(&out_file).unwrap(); - sccache_command() + client + .cmd() .args(compile_hip_cmdline( name, exe, @@ -1568,22 +1594,22 @@ fn test_hip_compiles_multi_targets(compiler: &Compiler, tempdir: &Path) { .success(); assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); trace!("request stats"); - get_stats(|info| { - assert_eq!(2, info.stats.compile_requests); - assert_eq!(2, info.stats.requests_executed); - assert_eq!(1, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("HIP").unwrap()); - assert_eq!(&1, info.stats.cache_misses.get("HIP").unwrap()); - let adv_hip_key = adv_key_kind("hip", compiler.name); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_hip_key).unwrap()); - assert_eq!(&1, info.stats.cache_misses.get_adv(&adv_hip_key).unwrap()); - }); + let stats = client.stats().unwrap(); + assert_eq!(2, stats.compile_requests); + assert_eq!(2, stats.requests_executed); + assert_eq!(1, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); + assert_eq!(&1, stats.cache_hits.get("HIP").unwrap()); + assert_eq!(&1, stats.cache_misses.get("HIP").unwrap()); + let adv_hip_key = adv_key_kind("hip", compiler.name); + assert_eq!(&1, stats.cache_hits.get_adv(&adv_hip_key).unwrap()); + assert_eq!(&1, stats.cache_misses.get_adv(&adv_hip_key).unwrap()); // By compiling another input source we verify that the pre-processor // phase is correctly running and outputting text trace!("compile B with gfx900 and gfx1030"); - sccache_command() + client + .cmd() .args(compile_hip_cmdline( name, exe, @@ -1598,28 +1624,27 @@ fn test_hip_compiles_multi_targets(compiler: &Compiler, tempdir: &Path) { .success(); assert!(fs::metadata(&out_file).map(|m| m.len() > 0).unwrap()); trace!("request stats"); - get_stats(|info| { - assert_eq!(3, info.stats.compile_requests); - assert_eq!(3, info.stats.requests_executed); - assert_eq!(1, info.stats.cache_hits.all()); - assert_eq!(2, info.stats.cache_misses.all()); - assert_eq!(&1, info.stats.cache_hits.get("HIP").unwrap()); - assert_eq!(&2, info.stats.cache_misses.get("HIP").unwrap()); - let adv_hip_key = adv_key_kind("hip", compiler.name); - assert_eq!(&1, info.stats.cache_hits.get_adv(&adv_hip_key).unwrap()); - assert_eq!(&2, info.stats.cache_misses.get_adv(&adv_hip_key).unwrap()); - }); + let stats = client.stats().unwrap(); + assert_eq!(3, stats.compile_requests); + assert_eq!(3, stats.requests_executed); + assert_eq!(1, stats.cache_hits.all()); + assert_eq!(2, stats.cache_misses.all()); + assert_eq!(&1, stats.cache_hits.get("HIP").unwrap()); + assert_eq!(&2, stats.cache_misses.get("HIP").unwrap()); + let adv_hip_key = adv_key_kind("hip", compiler.name); + assert_eq!(&1, stats.cache_hits.get_adv(&adv_hip_key).unwrap()); + assert_eq!(&2, stats.cache_misses.get_adv(&adv_hip_key).unwrap()); } -fn run_sccache_hip_command_tests(compiler: Compiler, tempdir: &Path) { - zero_stats(); - test_hip_compiles(&compiler, tempdir); - zero_stats(); - test_hip_compiles_multi_targets(&compiler, tempdir); +fn run_sccache_hip_command_tests(client: &SccacheClient, compiler: Compiler, tempdir: &Path) { + client.zero_stats(); + test_hip_compiles(client, &compiler, tempdir); + client.zero_stats(); + test_hip_compiles_multi_targets(client, &compiler, tempdir); // test_proper_lang_stat_tracking(compiler, tempdir); } -fn test_clang_multicall(compiler: Compiler, tempdir: &Path) { +fn test_clang_multicall(client: &SccacheClient, compiler: Compiler, tempdir: &Path) { let Compiler { name, exe, @@ -1630,7 +1655,8 @@ fn test_clang_multicall(compiler: Compiler, tempdir: &Path) { copy_to_tempdir(&[INPUT_CLANG_MULTICALL], tempdir); println!("compile clang_multicall"); - sccache_command() + client + .cmd() .args(compile_cmdline( name, exe, @@ -1645,6 +1671,7 @@ fn test_clang_multicall(compiler: Compiler, tempdir: &Path) { } fn test_clang_cache_whitespace_normalization( + client: &SccacheClient, compiler: Compiler, tempdir: &Path, hit: bool, @@ -1659,10 +1686,11 @@ fn test_clang_cache_whitespace_normalization( debug!("expecting hit: {}", hit); // Compile a source file. copy_to_tempdir(&[INPUT_WITH_WHITESPACE, INPUT_WITH_WHITESPACE_ALT], tempdir); - zero_stats(); + client.zero_stats(); debug!("compile whitespace"); - sccache_command() + client + .cmd() .args(compile_cmdline( name, &exe, @@ -1675,15 +1703,15 @@ fn test_clang_cache_whitespace_normalization( .assert() .success(); debug!("request stats"); - get_stats(|info| { - assert_eq!(1, info.stats.compile_requests); - assert_eq!(1, info.stats.requests_executed); - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - }); + let stats = client.stats().unwrap(); + assert_eq!(1, stats.compile_requests); + assert_eq!(1, stats.requests_executed); + assert_eq!(0, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); debug!("compile whitespace_alt"); - sccache_command() + client + .cmd() .args(compile_cmdline( name, &exe, @@ -1697,26 +1725,24 @@ fn test_clang_cache_whitespace_normalization( .success(); debug!("request stats (expecting cache hit)"); if hit { - get_stats(move |info| { - assert_eq!(2, info.stats.compile_requests); - assert_eq!(2, info.stats.requests_executed); - if preprocessor_cache_mode { - // Preprocessor cache mode hashes the input file, so whitespace - // normalization does not work. - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(2, info.stats.cache_misses.all()); - } else { - assert_eq!(1, info.stats.cache_hits.all()); - assert_eq!(1, info.stats.cache_misses.all()); - } - }); + let stats = client.stats().unwrap(); + assert_eq!(2, stats.compile_requests); + assert_eq!(2, stats.requests_executed); + if preprocessor_cache_mode { + // Preprocessor cache mode hashes the input file, so whitespace + // normalization does not work. + assert_eq!(0, stats.cache_hits.all()); + assert_eq!(2, stats.cache_misses.all()); + } else { + assert_eq!(1, stats.cache_hits.all()); + assert_eq!(1, stats.cache_misses.all()); + } } else { - get_stats(|info| { - assert_eq!(2, info.stats.compile_requests); - assert_eq!(2, info.stats.requests_executed); - assert_eq!(0, info.stats.cache_hits.all()); - assert_eq!(2, info.stats.cache_misses.all()); - }); + let stats = client.stats().unwrap(); + assert_eq!(2, stats.compile_requests); + assert_eq!(2, stats.requests_executed); + assert_eq!(0, stats.cache_hits.all()); + assert_eq!(2, stats.cache_misses.all()); } } @@ -1814,66 +1840,73 @@ fn find_hip_compiler() -> Option { None } +fn make_sccache_client( + preprocessor_cache_mode: bool, +) -> (Option, PathBuf, SccacheClient) { + let tempdir = tempfile::Builder::new() + .prefix("sccache_system_test") + .tempdir() + .unwrap(); + + // Persist the tempdir if SCCACHE_DEBUG is defined + let (tempdir_path, maybe_tempdir) = if env::var("SCCACHE_DEBUG").is_ok() { + (tempdir.into_path(), None) + } else { + (tempdir.path().to_path_buf(), Some(tempdir)) + }; + + // Create the configurations + let sccache_cfg = sccache_client_cfg(&tempdir_path, preprocessor_cache_mode); + write_json_cfg(&tempdir_path, "sccache-cfg.json", &sccache_cfg); + let sccache_cached_cfg_path = tempdir_path.join("sccache-cached-cfg"); + // Start the server daemon on a unique port + let client = SccacheClient::new( + &tempdir_path.join("sccache-cfg.json"), + &sccache_cached_cfg_path, + ) + .start(); + + (maybe_tempdir, tempdir_path, client) +} + // TODO: This runs multiple test cases, for multiple compilers. It should be // split up to run them individually. In the current form, it is hard to see // which sub test cases are executed, and if one fails, the remaining tests // are not run. #[test_case(true ; "with preprocessor cache")] #[test_case(false ; "without preprocessor cache")] -#[serial] #[cfg(any(unix, target_env = "msvc"))] fn test_sccache_command(preprocessor_cache_mode: bool) { let _ = env_logger::try_init(); - let tempdir = tempfile::Builder::new() - .prefix("sccache_system_test") - .tempdir() - .unwrap(); let compilers = find_compilers(); if compilers.is_empty() { - warn!("No compilers found, skipping test"); - } else { - // Ensure there's no existing sccache server running. - stop_local_daemon(); - // Create the configurations - let sccache_cfg = sccache_client_cfg(tempdir.path(), preprocessor_cache_mode); - write_json_cfg(tempdir.path(), "sccache-cfg.json", &sccache_cfg); - let sccache_cached_cfg_path = tempdir.path().join("sccache-cached-cfg"); - // Start a server. - trace!("start server"); - start_local_daemon( - &tempdir.path().join("sccache-cfg.json"), - &sccache_cached_cfg_path, - ); - for compiler in compilers { - run_sccache_command_tests(compiler, tempdir.path(), preprocessor_cache_mode); - zero_stats(); - } - stop_local_daemon(); + return warn!("No compilers found, skipping test"); + } + + // Create and start the sccache client + let (_tempdir, tempdir_path, client) = make_sccache_client(preprocessor_cache_mode); + + for compiler in compilers { + run_sccache_command_tests(&client, compiler, &tempdir_path, preprocessor_cache_mode); + client.zero_stats(); } } #[test] -#[serial] fn test_stats_no_server() { - // Ensure there's no existing sccache server running. - stop_local_daemon(); - get_stats(|_| {}); + let client = SccacheClient::new_no_cfg(); + let _ = client.stats(); assert!( - !stop_local_daemon(), + !client.stop(), "Server shouldn't be running after --show-stats" ); } #[test_case(true ; "with preprocessor cache")] #[test_case(false ; "without preprocessor cache")] -#[serial] #[cfg(any(unix, target_env = "msvc"))] fn test_cuda_sccache_command(preprocessor_cache_mode: bool) { let _ = env_logger::try_init(); - let tempdir = tempfile::Builder::new() - .prefix("sccache_system_test") - .tempdir() - .unwrap(); let compilers = find_cuda_compilers(); println!( "CUDA compilers: {:?}", @@ -1883,57 +1916,30 @@ fn test_cuda_sccache_command(preprocessor_cache_mode: bool) { .collect::>() ); if compilers.is_empty() { - warn!("No compilers found, skipping test"); - } else { - // Ensure there's no existing sccache server running. - stop_local_daemon(); - // Create the configurations - let sccache_cfg = sccache_client_cfg(tempdir.path(), preprocessor_cache_mode); - write_json_cfg(tempdir.path(), "sccache-cfg.json", &sccache_cfg); - let sccache_cached_cfg_path = tempdir.path().join("sccache-cached-cfg"); - // Start a server. - trace!("start server"); - start_local_daemon( - &tempdir.path().join("sccache-cfg.json"), - &sccache_cached_cfg_path, - ); - for compiler in compilers { - match compiler.name { - "nvcc" => run_sccache_nvcc_cuda_command_tests(compiler, tempdir.path()), - "clang++" => run_sccache_clang_cuda_command_tests(compiler, tempdir.path()), - _ => {} - } - zero_stats(); + return warn!("No compilers found, skipping test"); + } + + // Create and start the sccache client + let (_tempdir, tempdir_path, client) = make_sccache_client(preprocessor_cache_mode); + + for compiler in compilers { + match compiler.name { + "nvcc" => run_sccache_nvcc_cuda_command_tests(&client, compiler, &tempdir_path), + "clang++" => run_sccache_clang_cuda_command_tests(&client, compiler, &tempdir_path), + _ => {} } - stop_local_daemon(); } } #[test_case(true ; "with preprocessor cache")] #[test_case(false ; "without preprocessor cache")] -#[serial] #[cfg(any(unix, target_env = "msvc"))] fn test_hip_sccache_command(preprocessor_cache_mode: bool) { let _ = env_logger::try_init(); - let tempdir = tempfile::Builder::new() - .prefix("sccache_system_test") - .tempdir() - .unwrap(); if let Some(compiler) = find_hip_compiler() { - stop_local_daemon(); - // Create the configurations - let sccache_cfg = sccache_client_cfg(tempdir.path(), preprocessor_cache_mode); - write_json_cfg(tempdir.path(), "sccache-cfg.json", &sccache_cfg); - let sccache_cached_cfg_path = tempdir.path().join("sccache-cached-cfg"); - // Start a server. - trace!("start server"); - start_local_daemon( - &tempdir.path().join("sccache-cfg.json"), - &sccache_cached_cfg_path, - ); - run_sccache_hip_command_tests(compiler, tempdir.path()); - zero_stats(); - stop_local_daemon(); + // Create and start the sccache client + let (_tempdir, tempdir_path, client) = make_sccache_client(preprocessor_cache_mode); + run_sccache_hip_command_tests(&client, compiler, &tempdir_path); } }