Improvements to the config parser

docs/config.md

@@ -26,24 +26,23 @@ github:
   polling_interval: 15m
   webhook_secret: Some super secret text
 
-machine_snippets:
-  cfg-template: &cfg-template
-    setup_template:
-      path: /etc/forrest/templates/generic
-      parameters:
-        RUNNER_VERSION: "2.318.0"
-        RUNNER_HASH: "28ed88e4cedf0fc93201a901e392a70463dbd0213f2ce9d57a4ab495027f3e2f"
-
+.machines:
   os-arch: &os-arch
     base_image: /srv/forrest/images/Arch-Linux-x86_64-cloudimg.img
   os-debian: &os-debian
     base_image: /srv/forrest/images/debian-12-generic-amd64.raw
 
   machine-small: &machine-small
+    setup_template:
+      path: /etc/forrest/templates/generic
+      parameters:
+        RUNNER_VERSION: "2.318.0"
+        RUNNER_HASH: "28ed88e4cedf0fc93201a901e392a70463dbd0213f2ce9d57a4ab495027f3e2f"
     cpus: 4
     disk: 16G
     ram: 4G
   machine-medium: &machine-medium
+    << : *machine-small
     cpus: 8
     disk: 32G
     ram: 8G
@@ -54,20 +53,20 @@ repositories:
       persistence_token: <PERSISTENCE_TOKEN>
       machines:
         arch-base:
-          << : [*cfg-template, *os-arch, *machine-small]
+          << : [*os-arch, *machine-small]
           use_base: always
         debian-base:
-          << : [*cfg-template, *os-debian, *machine-small]
+          << : [*os-debian, *machine-small]
           use_base: always
         debian-yocto:
-          << : [*cfg-template, *os-debian, *machine-small]
+          << : [*os-debian, *machine-small]
           base_machine: hnez/forrest-images/debian-base
           use_base: always
 
     forrest-test:
       machines:
         test-debian:
-          << : [*cfg-template, *os-debian, *machine-medium]
+          << : [*os-debian, *machine-medium]
           base_machine: hnez/forrest-images/debian-base
 ```
 
@@ -112,11 +111,11 @@ Polling is a backup in case we have missed webhook events and is not a replacene
 for the webhook.
 The default interval is 15 minutes and should not be reduced too far.
 
-# `*_snippets`
+# `.*`
 
 (Optional)
 
-All top level configuration fields that end in `_snippets` are ignored.
+All configuration fields that start with a dot are ignored.
 These can be used to create config snippets that can be reused in other
 config sections.
 

src/config.rs

@@ -17,7 +17,7 @@ pub use github::GitHubConfig;
 pub use host::HostConfig;
 pub use machine::{Artifact, MachineConfig, NetworkInterface, Repository, SeedBasePolicy};
 
-#[derive(Deserialize)]
+#[derive(Debug, Deserialize, PartialEq)]
 #[serde(deny_unknown_fields)]
 pub struct ConfigFile {
     pub github: GitHubConfig,
@@ -36,34 +36,64 @@ pub struct Config {
     inner: Arc<Mutex<Inner>>,
 }
 
+fn contains_merge(value: &yaml_serde::Value) -> bool {
+    value
+        .as_mapping()
+        .map(|mapping| mapping.contains_key("<<") || mapping.values().any(contains_merge))
+        .unwrap_or(false)
+}
+
+fn remove_dot_keys(mapping: &mut yaml_serde::Mapping) {
+    // Remove all keys from the config that start with a dot.
+    // This is similar to how e.g. gitlab CI handles reusable YAML snippets.
+    mapping.retain(|k, _| k.as_str().map(|k| !k.starts_with(".")).unwrap_or(true));
+
+    // Recursively walk through all mappings in the config and remove
+    // dot prefixed keys there as well.
+    mapping
+        .values_mut()
+        .filter_map(yaml_serde::Value::as_mapping_mut)
+        .for_each(remove_dot_keys);
+}
+
 impl ConfigFile {
-    fn from_file(fd: &mut File) -> yaml_serde::Result<Arc<Self>> {
+    fn from_reader<R>(reader: R) -> yaml_serde::Result<Arc<Self>>
+    where
+        R: std::io::Read,
+    {
         // First we read the config file as generic yaml_serde Value.
-        let mut cfg: yaml_serde::Value = yaml_serde::from_reader(fd)?;
+        let mut cfg: yaml_serde::Value = yaml_serde::from_reader(reader)?;
 
         // Then we apply merges / overrides like these:
         //
-        // machine_snippets:
+        // .machines:
         //   small: &machine-small
         //     ram: 8G
         //     …
         //   large: &machine-large
         //     << : *machine-small
         //     ram: 32G
         //
-        cfg.apply_merge()?;
+        // We may need to do this multiple times, because `apply_merge` does
+        // not resolve nested merges by itself.
+        while contains_merge(&cfg) {
+            cfg.apply_merge()?;
+        }
 
         if let Some(cfg_mapping) = cfg.as_mapping_mut() {
             // Remove all top level fields from the config who's name ends
             // in `_snippets`.
             // This allows using keys like `machine_snippets` which do not
             // adhere to the syntax.
-
             cfg_mapping.retain(|k, _| {
                 k.as_str()
                     .map(|k| !k.ends_with("_snippets"))
                     .unwrap_or(true)
             });
+
+            // Recursively walk through all mappings in the config and remove
+            // dot prefixed keys.
+            remove_dot_keys(cfg_mapping);
         }
 
         // And then we convert to our config format.
@@ -96,7 +126,7 @@ impl Inner {
 
     fn get(&mut self) -> Arc<ConfigFile> {
         if let Some((mut fd, last_modified)) = self.should_refresh() {
-            match ConfigFile::from_file(&mut fd) {
+            match ConfigFile::from_reader(&mut fd) {
                 Ok(cf) => {
                     self.config_file = cf;
                     self.last_modified = last_modified;
@@ -116,7 +146,7 @@ impl Config {
     pub fn new<P: AsRef<Path>>(path: P) -> anyhow::Result<Self> {
         let mut fd = File::open(&path)?;
 
-        let config_file = ConfigFile::from_file(&mut fd)?;
+        let config_file = ConfigFile::from_reader(&mut fd)?;
         let last_modified = fd.metadata()?.modified()?;
 
         let inner = Inner {
@@ -140,3 +170,120 @@ impl Config {
         self.inner.lock().unwrap().get()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::ConfigFile;
+
+    const CONFIG_NESTED: &[u8] = br#"
+        host:
+          base_dir: /srv/forrest
+          ram: 120G
+
+        github:
+          app_id: 1234
+          jwt_key_file: key.pem
+          polling_interval: 15m
+          webhook_secret: Some super secret text
+
+        .machines:
+          machine-small: &machine-small
+            setup_template:
+              path: /etc/forrest/templates/generic
+              parameters:
+                RUNNER_VERSION: "2.318.0"
+                RUNNER_HASH: "28ed88e4cedf0fc93201a901e392a70463dbd0213f2ce9d57a4ab495027f3e2f"
+            base_image: /srv/forrest/images/debian-12-generic-amd64.raw
+            cpus: 4
+            disk: 16G
+            ram: 4G
+          machine-medium: &machine-medium
+            << : *machine-small
+            cpus: 8
+            disk: 32G
+            ram: 8G
+
+        repositories:
+          hnez:
+            forrest-images:
+              persistence_token: <PERSISTENCE_TOKEN>
+              machines:
+                debian-base:
+                  << : *machine-small
+                  use_base: always
+                debian-yocto:
+                  << : *machine-small
+                  base_machine: hnez/forrest-images/debian-base
+                  use_base: always
+
+            forrest-test:
+              machines:
+                test-debian:
+                  << : *machine-medium
+                  base_machine: hnez/forrest-images/debian-base
+        "#;
+
+    const CONFIG_FLAT: &[u8] = br#"
+        host:
+          base_dir: /srv/forrest
+          ram: 120G
+
+        github:
+          app_id: 1234
+          jwt_key_file: key.pem
+          polling_interval: 15m
+          webhook_secret: Some super secret text
+
+        repositories:
+          hnez:
+            forrest-images:
+              persistence_token: <PERSISTENCE_TOKEN>
+              machines:
+                debian-base:
+                  setup_template:
+                    path: /etc/forrest/templates/generic
+                    parameters:
+                      RUNNER_VERSION: "2.318.0"
+                      RUNNER_HASH: "28ed88e4cedf0fc93201a901e392a70463dbd0213f2ce9d57a4ab495027f3e2f"
+                  base_image: /srv/forrest/images/debian-12-generic-amd64.raw
+                  cpus: 4
+                  disk: 16G
+                  ram: 4G
+                  use_base: always
+                debian-yocto:
+                  setup_template:
+                    path: /etc/forrest/templates/generic
+                    parameters:
+                      RUNNER_VERSION: "2.318.0"
+                      RUNNER_HASH: "28ed88e4cedf0fc93201a901e392a70463dbd0213f2ce9d57a4ab495027f3e2f"
+                  base_image: /srv/forrest/images/debian-12-generic-amd64.raw
+                  cpus: 4
+                  disk: 16G
+                  ram: 4G
+                  base_machine: hnez/forrest-images/debian-base
+                  use_base: always
+
+            forrest-test:
+              machines:
+                test-debian:
+                  setup_template:
+                    path: /etc/forrest/templates/generic
+                    parameters:
+                      RUNNER_VERSION: "2.318.0"
+                      RUNNER_HASH: "28ed88e4cedf0fc93201a901e392a70463dbd0213f2ce9d57a4ab495027f3e2f"
+                  base_image: /srv/forrest/images/debian-12-generic-amd64.raw
+                  cpus: 8
+                  disk: 32G
+                  ram: 8G
+                  base_machine: hnez/forrest-images/debian-base
+
+        "#;
+
+    #[test]
+    fn nested_snippets() {
+        let config_file_nested = ConfigFile::from_reader(CONFIG_NESTED).unwrap();
+        let config_file_flat = ConfigFile::from_reader(CONFIG_FLAT).unwrap();
+
+        assert_eq!(config_file_nested, config_file_flat);
+    }
+}

src/config/github.rs

@@ -8,7 +8,7 @@ fn default_timeout() -> Duration {
     Duration::from_secs(15 * 60)
 }
 
-#[derive(Deserialize)]
+#[derive(Debug, Deserialize, PartialEq)]
 #[serde(deny_unknown_fields)]
 pub struct GitHubConfig {
     pub app_id: u64,

src/config/host.rs

@@ -4,7 +4,7 @@ use serde::Deserialize;
 
 use super::size_in_bytes::SizeInBytes;
 
-#[derive(Deserialize)]
+#[derive(Debug, Deserialize, PartialEq)]
 #[serde(deny_unknown_fields)]
 pub struct HostConfig {
     pub base_dir: PathBuf,

src/config/machine.rs

@@ -6,15 +6,15 @@ use serde::Deserialize;
 use super::size_in_bytes::SizeInBytes;
 use crate::machines::Triplet;
 
-#[derive(Deserialize)]
+#[derive(Debug, Deserialize, PartialEq)]
 pub struct SetupTemplate {
     pub path: PathBuf,
 
     #[serde(default)]
     pub parameters: HashMap<String, String>,
 }
 
-#[derive(Default, Deserialize, Clone, Copy)]
+#[derive(Debug, Default, Deserialize, Clone, Copy, PartialEq)]
 #[serde(rename_all = "snake_case")]
 pub enum SeedBasePolicy {
     #[default]
@@ -23,7 +23,7 @@ pub enum SeedBasePolicy {
     Never,
 }
 
-#[derive(Deserialize)]
+#[derive(Debug, Deserialize, PartialEq)]
 #[serde(deny_unknown_fields)]
 pub struct ExposedDirectory {
     pub path: PathBuf,
@@ -36,7 +36,7 @@ fn default_artifact_name() -> String {
     "artifact".into()
 }
 
-#[derive(Deserialize)]
+#[derive(Debug, Deserialize, PartialEq)]
 #[serde(deny_unknown_fields)]
 pub struct Artifact {
     #[serde(default = "default_artifact_name")]
@@ -47,21 +47,21 @@ pub struct Artifact {
     pub token: Option<String>,
 }
 
-#[derive(Deserialize)]
+#[derive(Debug, Deserialize, PartialEq)]
 #[serde(deny_unknown_fields)]
 pub struct NetworkInterfaceVde {
     pub path: PathBuf,
 }
 
-#[derive(Deserialize)]
+#[derive(Debug, Deserialize, PartialEq)]
 #[serde(tag = "type")]
 #[serde(deny_unknown_fields)]
 pub enum NetworkInterface {
     #[serde(rename = "vde")]
     Vde(NetworkInterfaceVde),
 }
 
-#[derive(Deserialize)]
+#[derive(Debug, Deserialize, PartialEq)]
 #[serde(deny_unknown_fields)]
 pub struct MachineConfig {
     pub base_machine: Option<Triplet>,
@@ -85,7 +85,7 @@ pub struct MachineConfig {
     pub network_interfaces: Vec<NetworkInterface>,
 }
 
-#[derive(Deserialize)]
+#[derive(Debug, Deserialize, PartialEq)]
 #[serde(deny_unknown_fields)]
 pub struct Repository {
     pub persistence_token: Option<String>,

src/config/size_in_bytes.rs

@@ -1,6 +1,6 @@
 use serde::{Deserialize, Deserializer};
 
-#[derive(Clone, Copy)]
+#[derive(Clone, Copy, Debug, PartialEq)]
 pub struct SizeInBytes(u64);
 
 impl<'de> Deserialize<'de> for SizeInBytes {