allow specifying custom stdout/err paths for jobs

Author: Panaetius

coman/src/cli.rs

@@ -31,6 +31,7 @@ pub enum CliCommands {
     },
 }
 
+#[allow(clippy::large_enum_variant)]
 #[derive(Subcommand, Debug)]
 pub enum CscsCommands {
     #[clap(about = "Log in to CSCS")]
@@ -87,6 +88,10 @@ pub enum CscsJobCommands {
         mount: Vec<(String, String)>,
         #[clap(short, long, help = "The docker image to use")]
         image: Option<DockerImageUrl>,
+        #[clap(short, long, help = "Path where stdout of the job gets written to")]
+        stdout: Option<PathBuf>,
+        #[clap(short, long, help = "Path where stderr of the job gets written to")]
+        stderr: Option<PathBuf>,
         #[clap(trailing_var_arg = true, help = "The command to run in the container")]
         command: Option<Vec<String>>,
     },

coman/src/cscs/api_client/client.rs

@@ -5,8 +5,8 @@ use eyre::eyre;
 use firecrest_client::{
     client::FirecrestClient,
     compute_api::{
-        cancel_compute_system_job, get_compute_system_job, get_compute_system_job_metadata, get_compute_system_jobs,
-        post_compute_system_job,
+        JobOptions, cancel_compute_system_job, get_compute_system_job, get_compute_system_job_metadata,
+        get_compute_system_jobs, post_compute_system_job,
     },
     filesystem_api::{
         get_filesystem_ops_download, get_filesystem_ops_ls, get_filesystem_ops_stat, get_filesystem_ops_tail,
@@ -22,8 +22,21 @@ use crate::{
     config::{ComputePlatform, Config},
     cscs::api_client::types::{FileStat, Job, JobDetail, PathEntry, S3Upload, System, UserInfo},
     trace_dbg,
+    util::types::DockerImageUrl,
 };
 
+#[derive(Debug, Clone, Default)]
+pub struct JobStartOptions {
+    pub script_file: Option<PathBuf>,
+    pub image: Option<DockerImageUrl>,
+    pub command: Option<Vec<String>>,
+    pub stdout: Option<PathBuf>,
+    pub stderr: Option<PathBuf>,
+    pub container_workdir: Option<String>,
+    pub env: Vec<(String, String)>,
+    pub mount: Vec<(String, String)>,
+}
+
 pub struct CscsApi {
     client: FirecrestClient,
 }
@@ -46,6 +59,7 @@ impl CscsApi {
         name: &str,
         script_path: PathBuf,
         envvars: HashMap<String, String>,
+        options: JobStartOptions,
     ) -> Result<()> {
         let workingdir = script_path.clone();
         let workingdir = workingdir.parent();
@@ -54,10 +68,14 @@ impl CscsApi {
             system_name,
             account,
             name,
-            None,
-            Some(script_path),
-            workingdir.map(|p| p.to_path_buf()),
-            envvars,
+            JobOptions {
+                script: None,
+                script_path: Some(script_path),
+                working_dir: workingdir.map(|p| p.to_path_buf()),
+                envvars,
+                stdout: options.stdout,
+                stderr: options.stderr,
+            },
         )
         .await?;
 
@@ -231,10 +249,14 @@ mod tests {
                     "",
                     None,
                     "",
-                    None,
-                    None,
-                    None,
-                    HashMap::new()
+                    JobOptions {
+                        script: None,
+                        script_path: None,
+                        working_dir: None,
+                        envvars: HashMap::new(),
+                        stdout: None,
+                        stderr: None
+                    }
                 ),
                 Result<PostJobSubmissionResponse>
             ))
@@ -243,7 +265,14 @@ mod tests {
                 Result<PostJobSubmissionResponse>
             ));
         let result = client
-            .start_job("test", None, "test", PathBuf::from("/test"), HashMap::new())
+            .start_job(
+                "test",
+                None,
+                "test",
+                PathBuf::from("/test"),
+                HashMap::new(),
+                JobStartOptions::default(),
+            )
             .await;
         assert_ok!(result);
     }

coman/src/cscs/cli.rs

@@ -18,13 +18,12 @@ use tokio::{
 use crate::{
     config::ComputePlatform,
     cscs::{
-        api_client::types::JobStatus,
+        api_client::{client::JobStartOptions, types::JobStatus},
         handlers::{
             cscs_file_download, cscs_file_list, cscs_file_upload, cscs_job_cancel, cscs_job_details, cscs_job_list,
             cscs_job_log, cscs_login, cscs_start_job, cscs_system_list, cscs_system_set,
         },
     },
-    util::types::DockerImageUrl,
 };
 
 pub(crate) async fn cli_cscs_login() -> Result<()> {
@@ -103,30 +102,12 @@ pub(crate) async fn cli_cscs_job_log(
 #[allow(clippy::too_many_arguments)]
 pub(crate) async fn cli_cscs_job_start(
     name: Option<String>,
-    script_file: Option<PathBuf>,
-    image: Option<DockerImageUrl>,
-    command: Option<Vec<String>>,
-    workdir: Option<String>,
-    env: Vec<(String, String)>,
-    mount: Vec<(String, String)>,
+    options: JobStartOptions,
     system: Option<String>,
     platform: Option<ComputePlatform>,
     account: Option<String>,
 ) -> Result<()> {
-    match cscs_start_job(
-        name,
-        script_file,
-        image,
-        command,
-        workdir,
-        env,
-        mount,
-        system,
-        platform,
-        account,
-    )
-    .await
-    {
+    match cscs_start_job(name, options, system, platform, account).await {
         Ok(_) => {
             println!("Job started");
             Ok(())

coman/src/cscs/handlers.rs

@@ -11,18 +11,15 @@ use crate::{
     config::{ComputePlatform, Config},
     cscs::{
         api_client::{
-            client::CscsApi,
+            client::{CscsApi, JobStartOptions},
             types::{FileStat, FileSystemType, Job, JobDetail, PathEntry, PathType, S3Upload, System, UserInfo},
         },
         oauth2::{
             CLIENT_ID_SECRET_NAME, CLIENT_SECRET_SECRET_NAME, client_credentials_login, finish_cscs_device_login,
             start_cscs_device_login,
         },
     },
-    util::{
-        keyring::{Secret, get_secret, store_secret},
-        types::DockerImageUrl,
-    },
+    util::keyring::{Secret, get_secret, store_secret},
 };
 
 const CSCS_MAX_DIRECT_SIZE: usize = 5242880;
@@ -169,12 +166,7 @@ pub async fn cscs_job_cancel(job_id: i64, system: Option<String>, platform: Opti
 #[allow(clippy::too_many_arguments)]
 pub async fn cscs_start_job(
     name: Option<String>,
-    script_file: Option<PathBuf>,
-    image: Option<DockerImageUrl>,
-    command: Option<Vec<String>>,
-    container_workdir: Option<String>,
-    env: Vec<(String, String)>,
-    mount: Vec<(String, String)>,
+    options: JobStartOptions,
     system: Option<String>,
     platform: Option<ComputePlatform>,
     account: Option<String>,
@@ -202,12 +194,15 @@ pub async fn cscs_start_job(
                     return Err(eyre!("couldn't get system description for {}", current_system));
                 }
             };
-            let container_workdir = container_workdir.unwrap_or(config.cscs.workdir.unwrap_or("/scratch".to_owned()));
+            let container_workdir = options
+                .container_workdir
+                .clone()
+                .unwrap_or(config.cscs.workdir.unwrap_or("/scratch".to_owned()));
             let base_path = scratch.join(user_info.name.clone()).join(&job_name);
 
             let mut envvars = config.cscs.env.clone();
-            envvars.extend(env);
-            let mut mount: HashMap<String, String> = mount.into_iter().collect();
+            envvars.extend(options.env.clone());
+            let mut mount: HashMap<String, String> = options.mount.clone().into_iter().collect();
             mount.entry("${SCRATCH}".to_owned()).or_insert("/scratch".to_owned());
 
             let mut tera = tera::Tera::default();
@@ -216,7 +211,7 @@ pub async fn cscs_start_job(
             let environment_template = config.cscs.edf_file_template;
             tera.add_raw_template("environment.toml", &environment_template)?;
 
-            let docker_image = image.unwrap_or(config.cscs.image.try_into()?);
+            let docker_image = options.image.clone().unwrap_or(config.cscs.image.try_into()?);
             let meta = docker_image.inspect().await?;
             if let Some(system_info) = config.cscs.systems.get(current_system) {
                 let mut compatible = false;
@@ -255,13 +250,18 @@ pub async fn cscs_start_job(
 
             // upload script
             let script_path = base_path.join("script.sh");
-            let script_template = script_file
+            let script_template = options
+                .script_file
+                .clone()
                 .map(std::fs::read_to_string)
                 .unwrap_or(Ok(config.cscs.sbatch_script_template))?;
             tera.add_raw_template("script.sh", &script_template)?;
             let mut context = tera::Context::new();
             context.insert("name", &job_name);
-            context.insert("command", &command.unwrap_or(config.cscs.command).join(" "));
+            context.insert(
+                "command",
+                &options.command.clone().unwrap_or(config.cscs.command).join(" "),
+            );
             context.insert("environment_file", &environment_path);
             context.insert("container_workdir", &container_workdir);
             let script = tera.render("script.sh", &context)?;
@@ -271,7 +271,7 @@ pub async fn cscs_start_job(
 
             // start job
             api_client
-                .start_job(current_system, account, &job_name, script_path, envvars)
+                .start_job(current_system, account, &job_name, script_path, envvars, options)
                 .await?;
             Ok(())
         }

coman/src/main.rs

@@ -24,6 +24,7 @@ use crate::{
     },
     config::Config,
     cscs::{
+        api_client::client::JobStartOptions,
         cli::{
             cli_cscs_file_download, cli_cscs_file_list, cli_cscs_file_upload, cli_cscs_job_cancel, cli_cscs_job_detail,
             cli_cscs_job_list, cli_cscs_job_log, cli_cscs_job_start, cli_cscs_login, cli_cscs_set_system,
@@ -78,15 +79,21 @@ async fn main() -> Result<()> {
                         workdir,
                         env,
                         mount,
+                        stdout,
+                        stderr,
                     } => {
                         cli_cscs_job_start(
                             name,
-                            script_file,
-                            image,
-                            command,
-                            workdir,
-                            env,
-                            mount,
+                            JobStartOptions {
+                                script_file,
+                                image,
+                                command,
+                                container_workdir: workdir,
+                                env,
+                                mount,
+                                stdout,
+                                stderr,
+                            },
                             system,
                             platform,
                             account,

firecrest_client/src/compute_api.rs

@@ -1,6 +1,6 @@
 use std::{collections::HashMap, path::PathBuf};
 
-use eyre::{Result, eyre};
+use eyre::{Result, WrapErr, eyre};
 use serde_json::json;
 
 use crate::{
@@ -11,34 +11,54 @@ use crate::{
     },
 };
 
-#[allow(clippy::too_many_arguments)]
-pub async fn post_compute_system_job(
+#[derive(Debug, Clone, Default)]
+pub struct JobOptions<'a> {
+    pub script: Option<&'a str>,
+    pub script_path: Option<PathBuf>,
+    pub working_dir: Option<PathBuf>,
+    pub envvars: HashMap<String, String>,
+    pub stdout: Option<PathBuf>,
+    pub stderr: Option<PathBuf>,
+}
+
+pub async fn post_compute_system_job<'a>(
     client: &FirecrestClient,
     system_name: &str,
     account: Option<String>,
     name: &str,
-    script: Option<&str>,
-    script_path: Option<PathBuf>,
-    working_dir: Option<PathBuf>,
-    envvars: HashMap<String, String>,
+    options: JobOptions<'a>,
 ) -> Result<PostJobSubmissionResponse> {
-    if script.is_none() && script_path.is_none() {
+    if options.script.is_none() && options.script_path.is_none() {
         return Err(eyre!("either script or script_path must be set"));
     }
     let body = PostJobSubmitRequest {
         job: JobDescriptionModel {
             name: Some(name.to_string()),
-            script: script.map(|s| s.to_owned()),
-            script_path: script_path
+            script: options.script.map(|s| s.to_owned()),
+            script_path: options
+                .script_path
                 .map(|s| s.into_os_string().into_string())
                 .transpose()
                 .map_err(|_| eyre!("couldn't convert script path"))?,
-            working_directory: working_dir
+            working_directory: options
+                .working_dir
                 .map(|s| s.into_os_string().into_string())
                 .transpose()
                 .map_err(|_| eyre!("couldn't convert working dir path"))?
                 .unwrap_or("/".to_owned()),
-            env: Some(JobDescriptionModelEnv::Object(json!(envvars))),
+            env: Some(JobDescriptionModelEnv::Object(json!(options.envvars))),
+            standard_output: options
+                .stdout
+                .map(|p| p.into_os_string().into_string())
+                .transpose()
+                .map_err(|e| eyre!("Path:{}", e.display()))
+                .wrap_err("stdout is not a valid path")?,
+            standard_error: options
+                .stderr
+                .map(|p| p.into_os_string().into_string())
+                .transpose()
+                .map_err(|e| eyre!("Path:{}", e.display()))
+                .wrap_err("stderr is not a valid path")?,
             account,
             ..Default::default()
         },