support config dropin directories

config files in acccording dropin directories are considered
automatically. The dropin directory is named as the config itself but
with a '.d' suffix. For example for local config the dropin directory is
'./west/config.d' and for global config it is '~/.westconfig.d'

Author: thorsten-klein

src/west/app/config.py

@@ -73,6 +73,14 @@
 
 To delete <name> everywhere it's set, including the system file:
     west config -D <name>
+
+Additionally to the config file a dropin config directory is considered.
+The directory is named as the according config file, but with a '.d' suffix.
+As a result there are three levels for dropin config directories (local, global
+and system), whereby all '.conf' files from each dropin directory are loaded in
+alphabetical order.
+For example:
+    .west/config.d/basics.conf
 '''
 
 CONFIG_EPILOG = '''\

src/west/configuration.py

@@ -68,63 +68,86 @@ def parse_key(dotted_name: str):
 
     @staticmethod
     def from_path(path: Path | None) -> '_InternalCF | None':
-        return _InternalCF(path) if path and path.exists() else None
+        cf = _InternalCF(path) if path else None
+        if not cf:
+            return None
+        if not cf.path and not cf.dropin_paths:
+            return None
+        return cf
 
     def __init__(self, path: Path):
-        self.path = path
         self.cp = _configparser()
-        read_files = self.cp.read(path, encoding='utf-8')
-        if len(read_files) != 1:
-            raise FileNotFoundError(path)
+        self.dropin_cp = _configparser()
+        self.path = path if path.exists() else None
+        dropin_dir = Path(f'{path}.d')
+        self.dropin_dir = dropin_dir if dropin_dir.exists() else None
+        self.dropin_paths = []
+        if self.dropin_dir:
+            # dropin configs are applied in alphabetical order
+            for conf in sorted(self.dropin_dir.iterdir()):
+                # only consider .conf files
+                if conf.suffix.lower() == '.conf':
+                    self.dropin_paths.append(self.dropin_dir / conf)
+        self._read()
+
+    def _read(self):
+        if self.path:
+            self.cp.read(self.path, encoding='utf-8')
+        if self.dropin_paths:
+            self.dropin_cp.read(self.dropin_paths, encoding='utf-8')
+
+    def _write(self):
+        if not self.path:
+            raise WestNotFound('No config file exists that can be written')
+        with open(self.path, 'w', encoding='utf-8') as f:
+            self.cp.write(f)
 
     def __contains__(self, option: str) -> bool:
         section, key = _InternalCF.parse_key(option)
 
-        return section in self.cp and key in self.cp[section]
+        if section in self.cp and key in self.cp[section]:
+            return True
+        return section in self.dropin_cp and key in self.dropin_cp[section]
 
     def get(self, option: str):
-        return self._get(option, self.cp.get)
+        return self._get(option, self.cp.get, self.dropin_cp.get)
 
     def getboolean(self, option: str):
-        return self._get(option, self.cp.getboolean)
+        return self._get(option, self.cp.getboolean, self.dropin_cp.getboolean)
 
     def getint(self, option: str):
-        return self._get(option, self.cp.getint)
+        return self._get(option, self.cp.getint, self.dropin_cp.getint)
 
     def getfloat(self, option: str):
-        return self._get(option, self.cp.getfloat)
+        return self._get(option, self.cp.getfloat, self.dropin_cp.getfloat)
 
-    def _get(self, option, getter):
+    def _get(self, option, config_getter, dropin_getter):
         section, key = _InternalCF.parse_key(option)
-
-        try:
-            return getter(section, key)
-        except (configparser.NoOptionError, configparser.NoSectionError) as err:
-            raise KeyError(option) from err
+        if section in self.cp and key in self.cp[section]:
+            getter = config_getter
+        elif section in self.dropin_cp and key in self.dropin_cp[section]:
+            getter = dropin_getter
+        else:
+            raise KeyError(option)
+        return getter(section, key)
 
     def set(self, option: str, value: Any):
         section, key = _InternalCF.parse_key(option)
-
         if section not in self.cp:
             self.cp[section] = {}
-
         self.cp[section][key] = value
-
-        with open(self.path, 'w', encoding='utf-8') as f:
-            self.cp.write(f)
+        self._write()
 
     def delete(self, option: str):
         section, key = _InternalCF.parse_key(option)
-
-        if section not in self.cp:
+        if (section not in self.cp) or (key not in self.cp[section]):
             raise KeyError(option)
 
         del self.cp[section][key]
         if not self.cp[section].items():
             del self.cp[section]
 
-        with open(self.path, 'w', encoding='utf-8') as f:
-            self.cp.write(f)
+        self._write()
 
 class ConfigFile(Enum):
     '''Types of west configuration file.
@@ -362,13 +385,14 @@ def _copy_to_configparser(self, cp: configparser.ConfigParser) -> None:
         # function-and-global-state APIs.
 
         def load(cf: _InternalCF):
-            for section, contents in cf.cp.items():
-                if section == 'DEFAULT':
-                    continue
-                if section not in cp:
-                    cp.add_section(section)
-                for key, value in contents.items():
-                    cp[section][key] = value
+            for cp in [cf.dropin_cp, cf.cp]:
+                for section, contents in cp.items():
+                    if section == 'DEFAULT':
+                        continue
+                    if section not in cp:
+                        cp.add_section(section)
+                    for key, value in contents.items():
+                        cp[section][key] = value
 
         if self._system:
             load(self._system)
@@ -415,11 +439,12 @@ def _cf_to_dict(cf: _InternalCF | None) -> dict[str, Any]:
         ret: dict[str, Any] = {}
         if cf is None:
             return ret
-        for section, contents in cf.cp.items():
-            if section == 'DEFAULT':
-                continue
-            for key, value in contents.items():
-                ret[f'{section}.{key}'] = value
+        for cp in [cf.dropin_cp, cf.cp]:
+            for section, contents in cp.items():
+                if section == 'DEFAULT':
+                    continue
+                for key, value in contents.items():
+                    ret[f'{section}.{key}'] = value
         return ret
 
 

tests/test_config.py

@@ -6,6 +6,7 @@
 import os
 import pathlib
 import subprocess
+import textwrap
 from typing import Any
 
 import pytest
@@ -208,6 +209,107 @@ def test_local_creation():
     assert 'pytest' not in cfg(f=GLOBAL)
     assert cfg(f=LOCAL)['pytest']['key'] == 'val'
 
+TEST_CASES_CONFIG_D = [
+    # (flag, env_var)
+    ('', 'WEST_CONFIG_LOCAL'),
+    ('--local', 'WEST_CONFIG_LOCAL'),
+    ('--system', 'WEST_CONFIG_SYSTEM'),
+    ('--global', 'WEST_CONFIG_GLOBAL'),
+]
+
+@pytest.mark.parametrize("test_case", TEST_CASES_CONFIG_D)
+def test_config_d_local(test_case):
+    flag, env_var = test_case
+    config_path = pathlib.Path(os.environ[env_var])
+    config_d_dir = pathlib.Path(f'{config_path}.d')
+    config_d_dir.mkdir()
+
+    # write value in actual config file
+    cmd(f'config {flag} pytest.key val')
+    cmd(f'config {flag} pytest.config-only val')
+
+    # read config value via command line
+    stdout = cmd(f'config {flag} pytest.key')
+    assert 'val' == stdout.rstrip()
+    stdout = cmd(f'config {flag} pytest.config-only')
+    assert 'val' == stdout.rstrip()
+
+    # read the config file
+    with open(config_path) as config_file:
+        config_file_initial = config_file.read()
+    assert config_file_initial == textwrap.dedent('''\
+        [pytest]
+        key = val
+        config-only = val
+
+        ''')
+
+    # create a dropin config under .d
+    with open(config_d_dir / 'a.conf', 'w') as conf:
+        conf.write(textwrap.dedent('''
+        [pytest]
+        key = from dropin a
+        dropin-only = from dropin a
+        dropin-only-a = from dropin a
+        '''))
+
+    # create a dropin config under .d
+    with open(config_d_dir / 'z.conf', 'w') as conf:
+        conf.write(textwrap.dedent('''
+        [pytest]
+        dropin-only = from dropin z
+        dropin-only-z = from dropin z
+        '''))
+
+    # value from config is prefered over dropin config
+    stdout = cmd(f'config {flag} pytest.key')
+    assert 'val' == stdout.rstrip()
+    stdout = cmd(f'config {flag} pytest.dropin-only-a')
+    assert 'from dropin a' == stdout.rstrip()
+    stdout = cmd(f'config {flag} pytest.dropin-only-z')
+    assert 'from dropin z' == stdout.rstrip()
+    # alphabetical order (z.conf is overwriting a.conf)
+    stdout = cmd(f'config {flag} pytest.dropin-only')
+    assert 'from dropin z' == stdout.rstrip()
+
+    # deletion of a value that is only set in a dropin config should fail
+    cmd_raises(f'config {flag} -d pytest.dropin-only', subprocess.CalledProcessError)
+
+    # deletion of a value that is set in config
+    stdout = cmd(f'config {flag} -d pytest.config-only')
+    assert '' == stdout
+    with open(config_path) as config_file:
+        config_file_edited = config_file.read()
+    assert config_file_edited == textwrap.dedent('''\
+        [pytest]
+        key = val
+
+        ''')
+
+    # remove config file
+    config_path.unlink()
+
+    # values from config are unset now
+    stderr = cmd_raises(f'config {flag} pytest.config-only', subprocess.CalledProcessError)
+    assert 'ERROR: pytest.config-only is unset' == stderr.rstrip()
+
+    # dropin applies now, since config does not exist
+    stdout = cmd(f'config {flag} pytest.key')
+    assert 'from dropin a' == stdout.rstrip()
+    # alphabetical order (z.conf is overwriting a.conf)
+    stdout = cmd(f'config {flag} pytest.dropin-only')
+    assert 'from dropin z' == stdout.rstrip()
+    # other values remain same
+    stdout = cmd(f'config {flag} pytest.dropin-only-a')
+    assert 'from dropin a' == stdout.rstrip()
+    # values specified in only one dropin config remain same
+    stdout = cmd(f'config {flag} pytest.dropin-only-z')
+    assert 'from dropin z' == stdout.rstrip()
+
+    # deletion of a value that is not existing anymore should fail
+    cmd_raises(f'config {flag} -d pytest.config-only', subprocess.CalledProcessError)
+
+
 def test_local_creation_with_topdir():
     # Like test_local_creation, with a specified topdir.