diff --git a/Cargo.lock b/Cargo.lock index 1faa574..c0eb94d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1062,6 +1062,7 @@ dependencies = [ "tokio", "tokio-util", "toml 0.9.7", + "toml_edit 0.23.9", "tracing", "tracing-error", "tracing-subscriber", @@ -4884,7 +4885,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ - "toml_edit 0.23.6", + "toml_edit 0.23.9", ] [[package]] @@ -6405,7 +6406,7 @@ dependencies = [ "indexmap 2.12.0", "serde_core", "serde_spanned 1.0.2", - "toml_datetime 0.7.2", + "toml_datetime 0.7.3", "toml_parser", "toml_writer", "winnow", @@ -6422,9 +6423,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f1085dec27c2b6632b04c80b3bb1b4300d6495d1e129693bdda7d91e72eec1" +checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" dependencies = [ "serde_core", ] @@ -6445,21 +6446,22 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.23.6" +version = "0.23.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3effe7c0e86fdff4f69cdd2ccc1b96f933e24811c5441d44904e8683e27184b" +checksum = "5d7cbc3b4b49633d57a0509303158ca50de80ae32c265093b24c414705807832" dependencies = [ "indexmap 2.12.0", - "toml_datetime 0.7.2", + "toml_datetime 0.7.3", "toml_parser", + "toml_writer", "winnow", ] [[package]] name = "toml_parser" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf893c33be71572e0e9aa6dd15e6677937abd686b066eac3f8cd3531688a627" +checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" dependencies = [ "winnow", ] @@ -6472,9 +6474,9 @@ checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] name = "toml_writer" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d163a63c116ce562a22cda521fcc4d79152e7aba014456fb5eb442f6d6a10109" +checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" [[package]] name = "tower" diff --git a/README.md b/README.md index cca38ca..c5d9b9e 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,24 @@ Compute Manager for managing HPC compute +Table of contents +================= + + + * [Installation](#installation) + * [Linux](#linux) + * [Macos](#macos) + * [Windows](#windows) + * [Usage](#usage) + * [Logging in](#logging-in) + * [CLI](#cli) + * [Terminal UI](#tui) + * [coman.toml config file](#comantoml-config-file) + * [Editing the config](#editing-the-config) + * [Development](#development) + * [Prerequisites](#prerequisites) + * [Install binaries](#install-binaries) + ## Installation @@ -188,6 +206,24 @@ Get the logs from a job coman cscs job log ``` +You can also manage files with coman. +List a remote directory: + +```shell +coman cscs file list /capstor/scratch/cscs/your_user +``` + +Download a file: + +```shell +coman cscs file download /capstor/scratch/cscs/your_user/your_file /local/target_file +``` + +Upload a file: + +```shell +coman cscs file upload /my/local/file /capstor/scratch/cscs/your_user/your_file +``` ### TUI @@ -199,7 +235,73 @@ The TUI should be pretty self-explanatory. It gives an overview of your jobs on refreshed every couple of seconds, lets you see the logs and all the other functionality of the CLI, just in an interactive way. +### coman.toml config file + +The config file options look as follows: + +```toml +name = "myproject" # the name of the project, used to generate job names + +[cscs] +# check https://docs.cscs.ch/access/firecrest/#firecrest-deployment-on-alps for possible system and platform combinations +current_system = "daint" # what system/cluster to execute commands on +current_platform = "HPC" # what platform to execute commands on (valid: HPC, ML or CW) + + +image = "ubuntu" # default docker image to use + +command = ["sleep", "1"] # command to execute within the container, i.e. the job you want to run + +# the sbatch script you want to execute +# this gets templated with values specified in the {{}} and {% %} expressions (see https://keats.github.io/tera/docs/#templates for +# more information on the template language). Note, this can also just be hardcoded without any template parameters. +# Available parameters: +# name: the name of the job +# environment_file: the path to the edf environment toml file in the cluster +# command: the command to run +# container_workdir: the working directory inside the container +sbatch_script_template = """ +#!/bin/bash +#SBATCH --job-name={{name}} +#SBATCH --ntasks=1 +#SBATCH --time=10:00 +srun {% if environment_file %}--environment={{environment_file}}{% endif %} {{command}} +""" + +# the edf environment toml file template +# this gets templated with values specified in the {{}} and {% %} expressions (see https://keats.github.io/tera/docs/#templates for +# more information on the template language). Note, this can also just be hardcoded without any template parameters. +# Available parameters: +# edf_image: the container image to use, in edf format +# container_workdir: the working directory to use within the container +# env: a dictionary of key/value pairs for environment variables to set in the container +# mount: a dictionary of key/value pairs for folders to mount to the container, with key being the path in the cluster and value being the path in the container +edf_file_template = """ +{% if edf_image %}image = "{{edf_image}}"{% endif %} +mounts = [{% for source, target in mount %}"{{source}}:{{target}}",{% endfor %}] +workdir = "{{container_workdir}}" + +[env] +{% for key, value in env %} +{{key}} = "{{value}}" +{% endfor %} +""" + +# set environment variables that should be passed to a job +[cscs.env] +ENV_VAR = "env_value" + +``` +#### Editing the config +You can edit the config file directly or (safer) use coman commands to do so: +```shell +coman config get cscs.current_system +``` + +```shell +coman config set cscs.current_system "daint" +``` ## Development @@ -235,4 +337,4 @@ If you want to use cargo to install `coman`, make sure to remove any version of ``` cargo install --path ./coman -``` \ No newline at end of file +``` diff --git a/coman/.config/config.toml b/coman/.config/config.toml index 7481b10..ae5e583 100644 --- a/coman/.config/config.toml +++ b/coman/.config/config.toml @@ -1,11 +1,21 @@ [cscs] -current_system = "daint" -current_platform = "HPC" +# check https://docs.cscs.ch/access/firecrest/#firecrest-deployment-on-alps for possible system and platform combinations +current_system = "daint" # what system/cluster to execute commands on +current_platform = "HPC" # what platform to execute commands on (valid: HPC, ML or CW) -image = "ubuntu" -command = ["sleep", "1"] +image = "ubuntu" # default docker image to use +command = ["sleep", "1"] # command to execute within the container, i.e. the job you want to run + +# the sbatch script you want to execute +# this gets templated with values specified in the {{}} and {% %} expressions (see https://keats.github.io/tera/docs/#templates for +# more information on the template language). Note, this can also just be hardcoded without any template parameters. +# Available parameters: +# name: the name of the job +# environment_file: the path to the edf environment toml file in the cluster +# command: the command to run +# container_workdir: the working directory inside the container sbatch_script_template = """ #!/bin/bash #SBATCH --job-name={{name}} @@ -14,6 +24,14 @@ sbatch_script_template = """ srun {% if environment_file %}--environment={{environment_file}}{% endif %} {{command}} """ +# the edf environment toml file template +# this gets templated with values specified in the {{}} and {% %} expressions (see https://keats.github.io/tera/docs/#templates for +# more information on the template language). Note, this can also just be hardcoded without any template parameters. +# Available parameters: +# edf_image: the container image to use, in edf format +# container_workdir: the working directory to use within the container +# env: a dictionary of key/value pairs for environment variables to set in the container +# mount: a dictionary of key/value pairs for folders to mount to the container, with key being the path in the cluster and value being the path in the container edf_file_template = """ {% if edf_image %}image = "{{edf_image}}"{% endif %} mounts = [{% for source, target in mount %}"{{source}}:{{target}}",{% endfor %}] @@ -25,8 +43,9 @@ workdir = "{{container_workdir}}" {% endfor %} """ +# set environment variables that should be passed to a job [cscs.env] - +# env_var = "env_value" [cscs.systems] @@ -35,3 +54,12 @@ architecture = ["arm64"] [cscs.systems.eiger] architecture = ["amd64"] + +[cscs.systems.bristen] +architecture = ["amd64"] + +[cscs.systems.clariden] +architecture = ["amd64"] + +[cscs.systems.santis] +architecture = ["arm64"] diff --git a/coman/Cargo.toml b/coman/Cargo.toml index 6338fb9..8d65971 100644 --- a/coman/Cargo.toml +++ b/coman/Cargo.toml @@ -72,6 +72,7 @@ chrono = "0.4.42" openssl = { version = "0.10.75", features = ["vendored"] } tui-realm-treeview = "3.0.0" aws-sdk-s3 = "1.115.0" +toml_edit = "0.23.9" [build-dependencies] anyhow = "1.0.90" diff --git a/coman/src/cli.rs b/coman/src/cli.rs index ca21c99..79b4f35 100644 --- a/coman/src/cli.rs +++ b/coman/src/cli.rs @@ -1,10 +1,11 @@ use std::{error::Error, path::PathBuf}; use clap::{Args, Parser, Subcommand, builder::TypedValueParser}; +use color_eyre::Result; use strum::VariantNames; use crate::{ - config::{ComputePlatform, get_config_dir, get_data_dir, get_project_local_config_file}, + config::{ComputePlatform, Config, get_config_dir, get_data_dir, get_project_local_config_file}, cscs::api_client::client::{EdfSpec as EdfSpecEnum, ScriptSpec as ScriptSpecEnum}, util::types::DockerImageUrl, }; @@ -27,8 +28,37 @@ pub enum CliCommands { }, #[clap(about = "Create a new project configuration file")] Init { - #[clap(help = "Destination folder to create config in (default = current directory)")] + #[clap(help = "destination folder to create config in (default = current directory)")] destination: Option, + #[clap(help = "project name to use")] + name: Option, + }, + #[clap(about = "Manage configuration")] + Config { + #[command(subcommand)] + command: ConfigCommands, + }, +} + +#[derive(Subcommand, Debug)] +pub enum ConfigCommands { + #[clap(about = "Set config values")] + Set { + #[clap( + short, + long, + action, + help = "whether to change the global config or the project local one" + )] + global: bool, + #[clap(help = "Config key path, e.g. `cscs.current_system`")] + key_path: String, + #[clap(help = "Value to set", value_parser = parse_toml_value)] + value: toml_edit::Value, + }, + Get { + #[clap(help = "Config key path, e.g. `cscs.current_system`")] + key_path: String, }, } @@ -251,6 +281,17 @@ Data directory: {data_dir_path}" ) } +pub fn set_config>(key_path: String, value: V, global: bool) -> Result<()> { + let mut config = Config::new()?; + config.set(&key_path, value, global)?; + Ok(()) +} + +pub fn get_config(key_path: String) -> Result { + let config = Config::new()?; + config.get(&key_path) +} + fn parse_key_val(s: &str) -> Result<(T, U), Box> where T: std::str::FromStr, @@ -275,3 +316,22 @@ where .ok_or_else(|| format!("invalid KEY:value: no `:` found in `{s}`"))?; Ok((s[..pos].parse()?, s[pos + 1..].parse()?)) } + +pub fn parse_toml_value(value_str: &str) -> Result { + match value_str.parse() { + Ok(value) => Ok(value), + Err(_) if is_bare_string(value_str) => Ok(value_str.into()), + Err(err) => Err(err), + } +} +fn is_bare_string(value_str: &str) -> bool { + // leading whitespace isn't ignored when parsing TOML value expression, but + // "\n[]" doesn't look like a bare string. + let trimmed = value_str.trim_ascii().as_bytes(); + if let (Some(&first), Some(&last)) = (trimmed.first(), trimmed.last()) { + // string, array, or table constructs? + !matches!(first, b'"' | b'\'' | b'[' | b'{') && !matches!(last, b'"' | b'\'' | b']' | b'}') + } else { + true // empty or whitespace only + } +} diff --git a/coman/src/components/status_bar.rs b/coman/src/components/status_bar.rs index 61c86e8..08a5417 100644 --- a/coman/src/components/status_bar.rs +++ b/coman/src/components/status_bar.rs @@ -42,8 +42,8 @@ impl StatusBar { last_updated: Instant::now(), current_status: None, status_clear_time: Duration::from_secs(10), - current_platform: config.cscs.current_platform.to_string(), - current_system: config.cscs.current_system, + current_platform: config.values.cscs.current_platform.to_string(), + current_system: config.values.cscs.current_system, } } } diff --git a/coman/src/config.rs b/coman/src/config.rs index c4af4e9..1b90182 100644 --- a/coman/src/config.rs +++ b/coman/src/config.rs @@ -2,26 +2,35 @@ use std::{collections::HashMap, env, path::PathBuf}; -use color_eyre::Result; +use color_eyre::{ + Result, + eyre::{Context, ContextCompat, eyre}, +}; use directories::ProjectDirs; -use eyre::eyre; use lazy_static::lazy_static; use serde::{Deserialize, Serialize}; use strum_macros::{EnumString, VariantNames}; +use toml_edit::DocumentMut; const DEFAULT_CONFIG_TOML: &str = include_str!("../.config/config.toml"); -#[derive(Clone, Debug, Serialize, Deserialize, Default)] -pub struct SystemDescription { - pub architecture: Vec, +const DEFAULT_KEYS: &[&str] = &["name", "cscs.account"]; + +lazy_static! { + pub static ref PROJECT_NAME: String = env!("CARGO_CRATE_NAME").to_uppercase().to_string(); + pub static ref DATA_FOLDER: Option = env::var(format!("{}_DATA", PROJECT_NAME.clone())) + .ok() + .map(PathBuf::from); + pub static ref CONFIG_FOLDER: Option = env::var(format!("{}_CONFIG", PROJECT_NAME.clone())) + .ok() + .map(PathBuf::from); + pub static ref CONFIG_FILE_NAME: String = format!("{}.toml", PROJECT_NAME.to_lowercase().clone()); + pub static ref CONFIG_FORMAT: config::FileFormat = config::FileFormat::Toml; } #[derive(Clone, Debug, Serialize, Deserialize, Default)] -pub struct AppConfig { - #[serde(default)] - pub data_dir: PathBuf, - #[serde(default)] - pub config_dir: PathBuf, +pub struct SystemDescription { + pub architecture: Vec, } #[derive(Clone, Debug, Serialize, Deserialize, Default, strum::Display, EnumString, VariantNames)] @@ -60,70 +69,153 @@ pub struct CscsConfig { } #[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct Config { +pub struct ComanConfig { #[serde(default)] pub name: Option, - #[serde(default, flatten)] - pub config: AppConfig, #[serde(default)] pub cscs: CscsConfig, } -lazy_static! { - pub static ref PROJECT_NAME: String = env!("CARGO_CRATE_NAME").to_uppercase().to_string(); - pub static ref DATA_FOLDER: Option = env::var(format!("{}_DATA", PROJECT_NAME.clone())) - .ok() - .map(PathBuf::from); - pub static ref CONFIG_FOLDER: Option = env::var(format!("{}_CONFIG", PROJECT_NAME.clone())) - .ok() - .map(PathBuf::from); - pub static ref CONFIG_FILE_NAME: String = format!("{}.toml", PROJECT_NAME.to_lowercase().clone()); - pub static ref CONFIG_FORMAT: config::FileFormat = config::FileFormat::Toml; +#[derive(Clone, Debug)] +pub struct Layer { + source: PathBuf, + data: DocumentMut, } -impl Config { - pub fn new() -> Result { - let builder = default_config_builder()?; - let builder = global_config_builder(builder)?; - let builder = project_local_config_builder(builder)?; - - let cfg: Self = builder.build()?.try_deserialize()?; - Ok(cfg) +impl Layer { + pub fn from_path(path: PathBuf) -> Result { + if !path.exists() { + return Ok(Self { + source: path, + data: DocumentMut::new(), + }); + } + if !path.is_file() { + return Err(eyre!("Config path {} is not a file", path.display())); + } + let content = + std::fs::read_to_string(path.clone()).wrap_err(format!("couldn't read config {}", path.display()))?; + let doc = content + .parse::() + .wrap_err(format!("couldn't parse toml file {}", path.display()))?; + Ok(Self { + source: path, + data: doc, + }) } - pub fn new_global() -> Result { - let builder = default_config_builder()?; - let builder = global_config_builder(builder)?; - let cfg: Self = builder.build()?.try_deserialize()?; - Ok(cfg) + pub fn get(&self, key_path: &str) -> Result> { + let key_path_parsed = toml_edit::Key::parse(key_path)?; + let root = self.data.as_item(); + let item = lookup_entry(key_path_parsed, root)?; + let item = item + .map(|i| i.clone().into_value()) + .transpose() + .map_err(|e| eyre!(format!("{:?}", e))) + .wrap_err("couldn't convert config item to value")?; + + Ok(item.map(|val| match val { + toml_edit::Value::String(v) => v.into_value(), + toml_edit::Value::Integer(_) + | toml_edit::Value::Float(_) + | toml_edit::Value::Boolean(_) + | toml_edit::Value::Datetime(_) + | toml_edit::Value::Array(_) + | toml_edit::Value::InlineTable(_) => val.decorated("", "").to_string(), + })) } - pub fn write_local(&self) -> Result<()> { - match get_project_local_config_file() { - Some(path) => { - let content = toml::to_string_pretty(self)?; - std::fs::write(path, content)?; - Ok(()) + pub fn set>(&mut self, key_path: &str, value: V) -> Result<()> { + let key_path_parsed = toml_edit::Key::parse(key_path)?; + let (leaf, keys) = key_path_parsed.split_last().wrap_err("couldn't parse key path")?; + let root_table: &mut dyn toml_edit::TableLike = self.data.as_table_mut(); + let table = keys + .iter() + .enumerate() + .try_fold(root_table, |table: &mut dyn toml_edit::TableLike, (i, key)| { + let sub_item = table.entry_format(key).or_insert_with(implicit_table); + sub_item.as_table_like_mut().ok_or(&keys[..=i]) + }) + .map_err(|e| eyre!("{:?}", e)) + .wrap_err("couldn't get config item path")?; + + match table.entry_format(leaf) { + toml_edit::Entry::Occupied(mut occupied_entry) => { + if !occupied_entry.get().is_value() { + return Err(eyre!("would overwrite entry {}", key_path)); + } + occupied_entry.insert(toml_edit::value(value)); + } + toml_edit::Entry::Vacant(vacant_entry) => { + vacant_entry.insert(toml_edit::value(value)); } - None => Err(eyre!( - "No config file exists in current project. Consider creating one using '{} init", - PROJECT_NAME.to_lowercase().clone() - )), } - } - pub fn write_global(&self) -> Result<()> { - let config_dir = get_config_dir(); - let path = config_dir.join(CONFIG_FILE_NAME.clone()); - let content = toml::to_string_pretty(self)?; - let parent = path.parent().unwrap(); - std::fs::create_dir_all(parent)?; - std::fs::write(path, content)?; Ok(()) } - pub fn create_config(destination: Option) -> Result<()> { - let mut config = Config::new()?; + pub fn write(&self) -> Result<()> { + let contents = self.data.to_string(); + std::fs::write(&self.source, contents).wrap_err("couldn't write config") + } +} + +fn lookup_entry(key_path_parsed: Vec, root: &toml_edit::Item) -> Result> { + let mut cur_item = root; + for key in key_path_parsed { + let Some(table) = cur_item.as_table_like() else { + return Err(eyre!("couldn't get subentry for {}", cur_item)); + }; + cur_item = match table.get(key.get()) { + Some(item) => item, + None => return Ok(None), + }; + } + Ok(Some(cur_item)) +} + +fn implicit_table() -> toml_edit::Item { + let mut table = toml_edit::Table::new(); + table.set_implicit(true); + toml_edit::Item::Table(table) +} + +#[derive(Clone, Debug)] +pub struct Config { + pub values: ComanConfig, + default_layer: toml_edit::DocumentMut, + global_layer: Layer, + project_layer: Option, +} + +impl Config { + pub fn new() -> Result { + let default_layer: DocumentMut = DEFAULT_CONFIG_TOML.parse()?; + let global_layer = global_config_layer()?; + let project_layer = project_local_config_layer()?; + let mut builder = + config::Config::builder().add_source(config::File::from_str(DEFAULT_CONFIG_TOML, config::FileFormat::Toml)); + builder = builder.add_source(config::File::from_str( + &global_layer.data.to_string(), + config::FileFormat::Toml, + )); + if let Some(project_layer) = project_layer.clone() { + builder = builder.add_source(config::File::from_str( + &project_layer.data.to_string(), + config::FileFormat::Toml, + )); + } + + let cfg: ComanConfig = builder.build()?.try_deserialize()?; + + Ok(Self { + values: cfg, + default_layer, + global_layer, + project_layer, + }) + } + pub fn create_project_config(destination: Option, name: Option) -> Result<()> { let project_dir = destination .unwrap_or(std::env::current_dir().expect("current directory does not exist")) .canonicalize()?; @@ -134,27 +226,90 @@ impl Config { )); } - let name = project_dir - .file_name() - .expect("could not get base name from destination"); - config.name = Some(name.to_string_lossy().to_string()); + let name = name.unwrap_or( + project_dir + .file_name() + .expect("could not get base name from destination") + .to_os_string() + .into_string() + .map_err(|e| eyre!("couldn't parse path: {:?}", e))?, + ); let config_path = project_dir.join(CONFIG_FILE_NAME.clone()); - std::fs::write(config_path.clone(), "")?; - let content = toml::to_string_pretty(&config)?; - std::fs::write(config_path, content)?; + let mut project_layer = Layer::from_path(config_path)?; + project_layer.set("name", name)?; + project_layer.write() + } + + pub fn set>(&mut self, key_path: &str, value: V, global: bool) -> Result<()> { + match global { + true => { + self.global_layer.set(key_path, value)?; + self.validate()?; + self.global_layer.write()?; + } + false => match self.project_layer { + Some(ref mut layer) => { + layer.set(key_path, value)?; + } + None => return Err(eyre!("No project config found, please create one with `coman init`")), + }, + }; + self.validate()?; + match global { + true => { + self.global_layer.write()?; + } + false => { + self.project_layer.as_ref().unwrap().write()?; + } + } + Ok(()) + } + + pub fn get(&self, key_path: &str) -> Result { + if let Some(ref layer) = self.project_layer { + match layer.get(key_path) { + Ok(Some(val)) => return Ok(val), + Ok(None) => {} + Err(e) => return Err(e), + } + } + + match self.global_layer.get(key_path) { + Ok(Some(val)) => return Ok(val), + Ok(None) => {} + Err(e) => return Err(e), + }; + + let key_path_parsed = toml_edit::Key::parse(key_path)?; + let item = lookup_entry(key_path_parsed, self.default_layer.as_item())?; + Ok(item.map(|i| i.to_string()).unwrap_or("".to_owned())) + } + + pub fn validate(&mut self) -> Result<()> { + let mut builder = + config::Config::builder().add_source(config::File::from_str(DEFAULT_CONFIG_TOML, config::FileFormat::Toml)); + builder = builder.add_source(config::File::from_str( + &self.global_layer.data.to_string(), + config::FileFormat::Toml, + )); + if let Some(project_layer) = self.project_layer.clone() { + builder = builder.add_source(config::File::from_str( + &project_layer.data.to_string(), + config::FileFormat::Toml, + )); + } + + let _cfg: ComanConfig = builder.build()?.try_deserialize().wrap_err("invalid config")?; Ok(()) } } -pub fn default_config_builder() -> Result> { - let data_dir = get_data_dir(); +pub fn global_config_layer() -> Result { let config_dir = get_config_dir(); - let builder = config::Config::builder() - .add_source(config::File::from_str(DEFAULT_CONFIG_TOML, config::FileFormat::Toml)) - .set_default("data_dir", data_dir.to_str().unwrap())? - .set_default("config_dir", config_dir.to_str().unwrap())?; - Ok(builder) + let source = config_dir.join(CONFIG_FILE_NAME.clone()); + Layer::from_path(source) } pub fn global_config_builder( @@ -168,6 +323,14 @@ pub fn global_config_builder( Ok(builder) } +pub fn project_local_config_layer() -> Result> { + if let Some(source) = get_project_local_config_file() { + let layer = Layer::from_path(source)?; + return Ok(Some(layer)); + } + Ok(None) +} + pub fn project_local_config_builder( builder: config::ConfigBuilder, ) -> Result> { diff --git a/coman/src/cscs/api_client/client.rs b/coman/src/cscs/api_client/client.rs index 6fdb162..2e4fda1 100644 --- a/coman/src/cscs/api_client/client.rs +++ b/coman/src/cscs/api_client/client.rs @@ -62,7 +62,7 @@ impl CscsApi { let client = FirecrestClient::default() .base_path(format!( "https://api.cscs.ch/{}/firecrest/v2/", - platform.unwrap_or(config.cscs.current_platform) + platform.unwrap_or(config.values.cscs.current_platform) ))? .token(token); Ok(Self { client }) diff --git a/coman/src/cscs/handlers.rs b/coman/src/cscs/handlers.rs index 1592170..7fc060b 100644 --- a/coman/src/cscs/handlers.rs +++ b/coman/src/cscs/handlers.rs @@ -77,16 +77,8 @@ pub async fn cscs_system_list(platform: Option) -> Result Result<()> { - if global { - let mut config = Config::new_global()?; - config.cscs.current_system = system_name; - config.write_global()?; - } else { - let mut config = Config::new()?; - config.cscs.current_system = system_name; - config.write_local()?; - } - Ok(()) + let mut config = Config::new()?; + config.set("cscs.current_system", system_name, global) } pub async fn cscs_job_list(system: Option, platform: Option) -> Result> { @@ -95,7 +87,7 @@ pub async fn cscs_job_list(system: Option, platform: Option Err(e), @@ -112,7 +104,7 @@ pub async fn cscs_job_details( let api_client = CscsApi::new(access_token.0, platform).unwrap(); let config = Config::new().unwrap(); api_client - .get_job(&system.unwrap_or(config.cscs.current_system), job_id) + .get_job(&system.unwrap_or(config.values.cscs.current_system), job_id) .await } Err(e) => Err(e), @@ -129,7 +121,7 @@ pub async fn cscs_job_log( Ok(access_token) => { let api_client = CscsApi::new(access_token.0, platform).unwrap(); let config = Config::new().unwrap(); - let current_system = &system.unwrap_or(config.cscs.current_system); + let current_system = &system.unwrap_or(config.values.cscs.current_system); let job = api_client.get_job(current_system, job_id).await?; if job.is_none() { return Err(eyre!("couldn't find job {}", job_id)); @@ -160,7 +152,7 @@ pub async fn cscs_job_cancel(job_id: i64, system: Option, platform: Opti let api_client = CscsApi::new(access_token.0, platform).unwrap(); let config = Config::new().unwrap(); api_client - .cancel_job(&system.unwrap_or(config.cscs.current_system), job_id) + .cancel_job(&system.unwrap_or(config.values.cscs.current_system), job_id) .await } Err(e) => Err(e), @@ -181,14 +173,17 @@ async fn handle_edf( EdfSpec::Generate => { let mut tera = tera::Tera::default(); - let environment_template = &config.cscs.edf_file_template; + let environment_template = &config.values.cscs.edf_file_template; tera.add_raw_template("environment.toml", environment_template)?; let mut mount: HashMap = options.mount.clone().into_iter().collect(); mount.entry("${SCRATCH}".to_owned()).or_insert("/scratch".to_owned()); - let docker_image = options.image.clone().unwrap_or(config.cscs.image.clone().try_into()?); + let docker_image = options + .image + .clone() + .unwrap_or(config.values.cscs.image.clone().try_into()?); let meta = docker_image.inspect().await?; - if let Some(system_info) = config.cscs.systems.get(current_system) { + if let Some(system_info) = config.values.cscs.systems.get(current_system) { let mut compatible = false; for sys_platform in system_info.architecture.iter() { if meta.platforms.contains(&sys_platform.clone().into()) { @@ -249,14 +244,14 @@ async fn handle_script( let script_path = base_path.join("script.sh"); match options.script_spec.clone() { ScriptSpec::Generate => { - let script_template = config.cscs.sbatch_script_template; + let script_template = config.values.cscs.sbatch_script_template; let mut tera = tera::Tera::default(); tera.add_raw_template("script.sh", &script_template)?; let mut context = tera::Context::new(); context.insert("name", &job_name); context.insert( "command", - &options.command.clone().unwrap_or(config.cscs.command).join(" "), + &options.command.clone().unwrap_or(config.values.cscs.command).join(" "), ); context.insert("environment_file", &environment_path.to_path_buf()); context.insert("container_workdir", &workdir); @@ -290,10 +285,12 @@ pub async fn cscs_job_start( Ok(access_token) => { let api_client = CscsApi::new(access_token.0, platform).unwrap(); let config = Config::new().unwrap(); - let current_system = &system.unwrap_or(config.cscs.current_system); - let account = account.or(config.cscs.account); + let current_system = &system.unwrap_or(config.values.cscs.current_system); + let account = account.or(config.values.cscs.account); let user_info = api_client.get_userinfo(current_system).await?; - let job_name = name.or(config.name).unwrap_or(format!("{}-coman", user_info.name)); + let job_name = name + .or(config.values.name) + .unwrap_or(format!("{}-coman", user_info.name)); let current_system_info = api_client.get_system(current_system).await?; let scratch = match current_system_info { Some(system) => PathBuf::from( @@ -312,10 +309,10 @@ pub async fn cscs_job_start( let container_workdir = options .container_workdir .clone() - .unwrap_or(config.cscs.workdir.unwrap_or("/scratch".to_owned())); + .unwrap_or(config.values.cscs.workdir.unwrap_or("/scratch".to_owned())); let base_path = scratch.join(user_info.name.clone()).join(&job_name); - let mut envvars = config.cscs.env.clone(); + let mut envvars = config.values.cscs.env.clone(); envvars.extend(options.env.clone()); let environment_path = handle_edf( @@ -359,7 +356,7 @@ pub async fn cscs_file_list( let api_client = CscsApi::new(access_token.0, platform).unwrap(); let config = Config::new().unwrap(); api_client - .list_path(&system.unwrap_or(config.cscs.current_system), path) + .list_path(&system.unwrap_or(config.values.cscs.current_system), path) .await } Err(e) => Err(e), @@ -382,7 +379,7 @@ pub async fn cscs_file_download( Ok(access_token) => { let api_client = CscsApi::new(access_token.0, platform).unwrap(); let config = Config::new().unwrap(); - let current_system = &system.unwrap_or(config.cscs.current_system); + let current_system = &system.unwrap_or(config.values.cscs.current_system); let paths = api_client.list_path(current_system, remote.clone()).await?; let path = paths.first().ok_or(eyre!("remote path doesn't exist"))?; if let PathType::Directory = path.path_type { @@ -396,7 +393,7 @@ pub async fn cscs_file_download( Ok(None) } else { // download via s3 - let account = account.or(config.cscs.account); + let account = account.or(config.values.cscs.account); let job_data = api_client.transfer_download(current_system, account, remote).await?; Ok(Some((job_data.0, job_data.1, size))) } @@ -415,7 +412,7 @@ pub async fn cscs_file_upload( Ok(access_token) => { let api_client = CscsApi::new(access_token.0, platform).unwrap(); let config = Config::new().unwrap(); - let current_system = &system.unwrap_or(config.cscs.current_system); + let current_system = &system.unwrap_or(config.values.cscs.current_system); let existing = api_client.list_path(current_system, remote.clone()).await?; let remote = if !existing.is_empty() { if existing.len() == 1 && existing[0].path_type == PathType::File { @@ -441,7 +438,7 @@ pub async fn cscs_file_upload( Ok(None) } else { // upload via s3 - let account = account.or(config.cscs.account); + let account = account.or(config.values.cscs.account); let transfer_data = api_client .transfer_upload(current_system, account, remote, size as i64) .await?; @@ -463,7 +460,7 @@ pub async fn cscs_stat_path( let api_client = CscsApi::new(access_token.0, platform).unwrap(); let config = Config::new().unwrap(); api_client - .stat_path(&system.unwrap_or(config.cscs.current_system), path) + .stat_path(&system.unwrap_or(config.values.cscs.current_system), path) .await } Err(e) => Err(e), @@ -476,7 +473,7 @@ pub async fn cscs_user_info(system: Option, platform: Option Err(e), diff --git a/coman/src/cscs/ports.rs b/coman/src/cscs/ports.rs index 5ee7439..9426f17 100644 --- a/coman/src/cscs/ports.rs +++ b/coman/src/cscs/ports.rs @@ -251,8 +251,8 @@ async fn list_files(id: PathBuf) -> Result>> { let systems = cscs_system_list(None).await?; let system = systems .iter() - .find(|s| s.name == config.cscs.current_system) - .unwrap_or_else(|| panic!("couldn't get info for system {}", config.cscs.current_system)); + .find(|s| s.name == config.values.cscs.current_system) + .unwrap_or_else(|| panic!("couldn't get info for system {}", config.values.cscs.current_system)); // listing big directories fails in the api and we might not actually be allowed to // access the roots of the storage. // So we try to append the user name to the paths and use that, if it works diff --git a/coman/src/main.rs b/coman/src/main.rs index ce9b0e5..5357eed 100644 --- a/coman/src/main.rs +++ b/coman/src/main.rs @@ -17,7 +17,7 @@ use crate::{ model::Model, user_events::{CscsEvent, FileEvent, StatusEvent, UserEvent}, }, - cli::{Cli, version}, + cli::{Cli, get_config, set_config, version}, components::{ file_tree::FileTree, global_listener::GlobalListener, status_bar::StatusBar, toolbar::Toolbar, workload_list::WorkloadList, @@ -58,6 +58,16 @@ async fn main() -> Result<()> { match args.command { Some(command) => match command { cli::CliCommands::Version => println!("{}", version()), + cli::CliCommands::Config { + command: config_command, + } => match config_command { + cli::ConfigCommands::Set { + key_path, + value, + global, + } => set_config(key_path, value, global)?, + cli::ConfigCommands::Get { key_path } => println!("{}", get_config(key_path)?), + }, cli::CliCommands::Cscs { command: cscs_command, system, @@ -120,7 +130,7 @@ async fn main() -> Result<()> { } }, }, - cli::CliCommands::Init { destination } => Config::create_config(destination)?, + cli::CliCommands::Init { destination, name } => Config::create_project_config(destination, name)?, }, None => run_tui(args.tick_rate)?, }