rework config management (#43)

* rework config management to allow better updates

* update readme

Author: Panaetius

Cargo.lock

@@ -5,6 +5,24 @@
 
 Compute Manager for managing HPC compute
 
+Table of contents
+=================
+
+<!--ts-->
+   * [Installation](#installation)
+      * [Linux](#linux)
+      * [Macos](#macos)
+      * [Windows](#windows)
+   * [Usage](#usage)
+      * [Logging in](#logging-in)
+      * [CLI](#cli)
+      * [Terminal UI](#tui)
+      * [coman.toml config file](#comantoml-config-file)
+        * [Editing the config](#editing-the-config)
+   * [Development](#development)
+     * [Prerequisites](#prerequisites)
+     * [Install binaries](#install-binaries)
+<!--te-->
 
 ## Installation
 
@@ -188,6 +206,24 @@ Get the logs from a job
 coman cscs job log <id>
 ```
 
+You can also manage files with coman.
+List a remote directory:
+
+```shell
+coman cscs file list /capstor/scratch/cscs/your_user
+```
+
+Download a file:
+
+```shell
+coman cscs file download /capstor/scratch/cscs/your_user/your_file /local/target_file
+```
+
+Upload a file:
+
+```shell
+coman cscs file upload /my/local/file /capstor/scratch/cscs/your_user/your_file
+```
 
 ### TUI
 
@@ -199,7 +235,73 @@ The TUI should be pretty self-explanatory. It gives an overview of your jobs on
 refreshed every couple of seconds, lets you see the logs and all the other functionality of the CLI,
 just in an interactive way.
 
+### coman.toml config file
+
+The config file options look as follows:
+
+```toml
+name = "myproject" # the name of the project, used to generate job names
+
+[cscs]
+# check https://docs.cscs.ch/access/firecrest/#firecrest-deployment-on-alps for possible system and platform combinations
+current_system = "daint" # what system/cluster to execute commands on
+current_platform = "HPC" # what platform to execute commands on (valid: HPC, ML or CW)
+
+
+image = "ubuntu" # default docker image to use
+
+command = ["sleep", "1"] # command to execute within the container, i.e. the job you want to run
+
+# the sbatch script you want to execute
+# this gets templated with values specified in the {{}} and {% %} expressions (see https://keats.github.io/tera/docs/#templates for
+# more information on the template language). Note, this can also just be hardcoded without any template parameters.
+# Available parameters:
+#   name: the name of the job
+#   environment_file: the path to the edf environment toml file in the cluster
+#   command: the command to run
+#   container_workdir: the working directory inside the container
+sbatch_script_template = """
+#!/bin/bash
+#SBATCH --job-name={{name}}
+#SBATCH --ntasks=1
+#SBATCH --time=10:00
+srun {% if environment_file %}--environment={{environment_file}}{% endif %} {{command}}
+"""
+
+# the edf environment toml file template
+# this gets templated with values specified in the {{}} and {% %} expressions (see https://keats.github.io/tera/docs/#templates for
+# more information on the template language). Note, this can also just be hardcoded without any template parameters.
+# Available parameters:
+#   edf_image: the container image to use, in edf format
+#   container_workdir: the working directory to use within the container
+#   env: a dictionary of key/value pairs for environment variables to set in the container
+#   mount: a dictionary of key/value pairs for folders to mount to the container, with key being the path in the cluster and value being the path in the container
+edf_file_template = """
+{% if edf_image %}image = "{{edf_image}}"{% endif %}
+mounts = [{% for source, target in mount %}"{{source}}:{{target}}",{% endfor %}]
+workdir = "{{container_workdir}}"
+
+[env]
+{% for key, value in env %}
+{{key}} = "{{value}}"
+{% endfor %}
+"""
+
+# set environment variables that should be passed to a job
+[cscs.env]
+ENV_VAR = "env_value"
+
+```
+#### Editing the config
 
+You can edit the config file directly or (safer) use coman commands to do so:
+```shell
+coman config get cscs.current_system
+```
+
+```shell
+coman config set cscs.current_system "daint"
+```
 
 ## Development
 
@@ -235,4 +337,4 @@ If you want to use cargo to install `coman`, make sure to remove any version of
 
 ```
 cargo install --path ./coman
-```
+```

README.md

@@ -1,11 +1,21 @@
 [cscs]
-current_system = "daint"
-current_platform = "HPC"
+# check https://docs.cscs.ch/access/firecrest/#firecrest-deployment-on-alps for possible system and platform combinations
+current_system = "daint" # what system/cluster to execute commands on
+current_platform = "HPC" # what platform to execute commands on (valid: HPC, ML or CW)
 
-image = "ubuntu"
 
-command = ["sleep", "1"]
+image = "ubuntu" # default docker image to use
 
+command = ["sleep", "1"] # command to execute within the container, i.e. the job you want to run
+
+# the sbatch script you want to execute
+# this gets templated with values specified in the {{}} and {% %} expressions (see https://keats.github.io/tera/docs/#templates for
+# more information on the template language). Note, this can also just be hardcoded without any template parameters.
+# Available parameters:
+#   name: the name of the job
+#   environment_file: the path to the edf environment toml file in the cluster
+#   command: the command to run
+#   container_workdir: the working directory inside the container
 sbatch_script_template = """
 #!/bin/bash
 #SBATCH --job-name={{name}}
@@ -14,6 +24,14 @@ sbatch_script_template = """
 srun {% if environment_file %}--environment={{environment_file}}{% endif %} {{command}}
 """
 
+# the edf environment toml file template
+# this gets templated with values specified in the {{}} and {% %} expressions (see https://keats.github.io/tera/docs/#templates for
+# more information on the template language). Note, this can also just be hardcoded without any template parameters.
+# Available parameters:
+#   edf_image: the container image to use, in edf format
+#   container_workdir: the working directory to use within the container
+#   env: a dictionary of key/value pairs for environment variables to set in the container
+#   mount: a dictionary of key/value pairs for folders to mount to the container, with key being the path in the cluster and value being the path in the container
 edf_file_template = """
 {% if edf_image %}image = "{{edf_image}}"{% endif %}
 mounts = [{% for source, target in mount %}"{{source}}:{{target}}",{% endfor %}]
@@ -25,8 +43,9 @@ workdir = "{{container_workdir}}"
 {% endfor %}
 """
 
+# set environment variables that should be passed to a job
 [cscs.env]
-
+# env_var = "env_value"
 
 [cscs.systems]
 
@@ -35,3 +54,12 @@ architecture = ["arm64"]
 
 [cscs.systems.eiger]
 architecture = ["amd64"]
+
+[cscs.systems.bristen]
+architecture = ["amd64"]
+
+[cscs.systems.clariden]
+architecture = ["amd64"]
+
+[cscs.systems.santis]
+architecture = ["arm64"]

coman/.config/config.toml

@@ -72,6 +72,7 @@ chrono = "0.4.42"
 openssl = { version = "0.10.75", features = ["vendored"] }
 tui-realm-treeview = "3.0.0"
 aws-sdk-s3 = "1.115.0"
+toml_edit = "0.23.9"
 
 [build-dependencies]
 anyhow = "1.0.90"

coman/Cargo.toml

@@ -1,10 +1,11 @@
 use std::{error::Error, path::PathBuf};
 
 use clap::{Args, Parser, Subcommand, builder::TypedValueParser};
+use color_eyre::Result;
 use strum::VariantNames;
 
 use crate::{
-    config::{ComputePlatform, get_config_dir, get_data_dir, get_project_local_config_file},
+    config::{ComputePlatform, Config, get_config_dir, get_data_dir, get_project_local_config_file},
     cscs::api_client::client::{EdfSpec as EdfSpecEnum, ScriptSpec as ScriptSpecEnum},
     util::types::DockerImageUrl,
 };
@@ -27,8 +28,37 @@ pub enum CliCommands {
     },
     #[clap(about = "Create a new project configuration file")]
     Init {
-        #[clap(help = "Destination folder to create config in (default = current directory)")]
+        #[clap(help = "destination folder to create config in (default = current directory)")]
         destination: Option<PathBuf>,
+        #[clap(help = "project name to use")]
+        name: Option<String>,
+    },
+    #[clap(about = "Manage configuration")]
+    Config {
+        #[command(subcommand)]
+        command: ConfigCommands,
+    },
+}
+
+#[derive(Subcommand, Debug)]
+pub enum ConfigCommands {
+    #[clap(about = "Set config values")]
+    Set {
+        #[clap(
+            short,
+            long,
+            action,
+            help = "whether to change the global config or the project local one"
+        )]
+        global: bool,
+        #[clap(help = "Config key path, e.g. `cscs.current_system`")]
+        key_path: String,
+        #[clap(help = "Value to set", value_parser = parse_toml_value)]
+        value: toml_edit::Value,
+    },
+    Get {
+        #[clap(help = "Config key path, e.g. `cscs.current_system`")]
+        key_path: String,
     },
 }
 
@@ -251,6 +281,17 @@ Data directory: {data_dir_path}"
     )
 }
 
+pub fn set_config<V: Into<toml_edit::Value>>(key_path: String, value: V, global: bool) -> Result<()> {
+    let mut config = Config::new()?;
+    config.set(&key_path, value, global)?;
+    Ok(())
+}
+
+pub fn get_config(key_path: String) -> Result<String> {
+    let config = Config::new()?;
+    config.get(&key_path)
+}
+
 fn parse_key_val<T, U>(s: &str) -> Result<(T, U), Box<dyn Error + Send + Sync + 'static>>
 where
     T: std::str::FromStr,
@@ -275,3 +316,22 @@ where
         .ok_or_else(|| format!("invalid KEY:value: no `:` found in `{s}`"))?;
     Ok((s[..pos].parse()?, s[pos + 1..].parse()?))
 }
+
+pub fn parse_toml_value(value_str: &str) -> Result<toml_edit::Value, toml_edit::TomlError> {
+    match value_str.parse() {
+        Ok(value) => Ok(value),
+        Err(_) if is_bare_string(value_str) => Ok(value_str.into()),
+        Err(err) => Err(err),
+    }
+}
+fn is_bare_string(value_str: &str) -> bool {
+    // leading whitespace isn't ignored when parsing TOML value expression, but
+    // "\n[]" doesn't look like a bare string.
+    let trimmed = value_str.trim_ascii().as_bytes();
+    if let (Some(&first), Some(&last)) = (trimmed.first(), trimmed.last()) {
+        // string, array, or table constructs?
+        !matches!(first, b'"' | b'\'' | b'[' | b'{') && !matches!(last, b'"' | b'\'' | b']' | b'}')
+    } else {
+        true // empty or whitespace only
+    }
+}

coman/src/cli.rs

@@ -42,8 +42,8 @@ impl StatusBar {
             last_updated: Instant::now(),
             current_status: None,
             status_clear_time: Duration::from_secs(10),
-            current_platform: config.cscs.current_platform.to_string(),
-            current_system: config.cscs.current_system,
+            current_platform: config.values.cscs.current_platform.to_string(),
+            current_system: config.values.cscs.current_system,
         }
     }
 }