support multiple west config files per config level

multiple config files can be specified in the according environment
variable for each config level (separated by 'os.pathsep', which is ';'
on Windows or ':' otherwise). The config files are applied in the same
order as they are specified, whereby values from later files override
earlier ones.

Author: thorsten-klein

src/west/app/config.py

@@ -49,6 +49,11 @@
 providing one of the following arguments:
     --local | --system | --global
 
+For each configuration level (local, global, and system) also multiple config
+file locations can be specified. To do so, set according environment variable
+to contain all paths (separated by 'os.pathsep', which is ';' on Windows or
+':' otherwise): Latter configuration files have precedence in such lists.
+
 The following command prints a list of all configuration files currently
 considered and existing (listed in the order as they are loaded):
     west config --list-paths

src/west/configuration.py

@@ -44,7 +44,7 @@
 from pathlib import Path, PureWindowsPath
 from typing import TYPE_CHECKING, Any
 
-from west.util import WEST_DIR, PathType, WestNotFound, west_dir
+from west.util import WEST_DIR, PathType, WestNotFound, paths_to_str, west_dir
 
 
 class MalformedConfig(Exception):
@@ -70,19 +70,36 @@ def parse_key(dotted_name: str):
         return section_child
 
     @staticmethod
-    def from_path(path: Path | None) -> '_InternalCF | None':
-        return _InternalCF(path) if path and path.exists() else None
-
-    def __init__(self, path: Path):
-        self.path = path
+    def from_path(path: Path | list[Path] | None) -> '_InternalCF | None':
+        if not path:
+            return None
+
+        # Normalize to a list of existing paths
+        paths = path
+        if isinstance(paths, Path):
+            paths = [Path(p) for p in str(paths).split(os.pathsep)]
+        paths = [p for p in paths if p.exists()]
+        return _InternalCF(paths) if paths else None
+
+    def __init__(self, paths: list[Path]):
         self.cp = _configparser()
-        read_files = self.cp.read(path, encoding='utf-8')
-        if len(read_files) != 1:
-            raise FileNotFoundError(path)
+        self.paths = paths
+        read_files = self.cp.read(self.paths, encoding='utf-8')
+        if len(read_files) != len(self.paths):
+            raise WestNotFound(f"Error while reading one of '{paths_to_str(paths)}'")
+
+    def _write(self):
+        if not self.paths:
+            raise WestNotFound('No config file exists that can be written')
+        if len(self.paths) > 1:
+            raise WestNotFound(
+                f'Cannot write if multiple configs in use: {paths_to_str(self.paths)}'
+            )
+        with open(self.paths[0], 'w', encoding='utf-8') as f:
+            self.cp.write(f)
 
     def __contains__(self, option: str) -> bool:
         section, key = _InternalCF.parse_key(option)
-
         return section in self.cp and key in self.cp[section]
 
     def get(self, option: str):
@@ -97,37 +114,31 @@ def getint(self, option: str):
     def getfloat(self, option: str):
         return self._get(option, self.cp.getfloat)
 
-    def _get(self, option, getter):
+    def _get(self, option, config_getter):
         section, key = _InternalCF.parse_key(option)
-
-        try:
-            return getter(section, key)
-        except (configparser.NoOptionError, configparser.NoSectionError) as err:
-            raise KeyError(option) from err
+        if section in self.cp and key in self.cp[section]:
+            getter = config_getter
+        else:
+            raise KeyError(option)
+        return getter(section, key)
 
     def set(self, option: str, value: Any):
         section, key = _InternalCF.parse_key(option)
-
         if section not in self.cp:
             self.cp[section] = {}
-
         self.cp[section][key] = value
-
-        with open(self.path, 'w', encoding='utf-8') as f:
-            self.cp.write(f)
+        self._write()
 
     def delete(self, option: str):
         section, key = _InternalCF.parse_key(option)
-
-        if section not in self.cp:
+        if option not in self:
             raise KeyError(option)
 
         del self.cp[section][key]
         if not self.cp[section].items():
             del self.cp[section]
 
-        with open(self.path, 'w', encoding='utf-8') as f:
-            self.cp.write(f)
+        self._write()
 
 
 class ConfigFile(Enum):
@@ -186,11 +197,11 @@ def __init__(self, topdir: PathType | None = None):
     def get_paths(self, location: ConfigFile = ConfigFile.ALL):
         ret = []
         if self._global and location in [ConfigFile.GLOBAL, ConfigFile.ALL]:
-            ret.append(self._global.path)
+            ret.extend(self._global.paths)
         if self._system and location in [ConfigFile.SYSTEM, ConfigFile.ALL]:
-            ret.append(self._system.path)
+            ret.extend(self._system.paths)
         if self._local and location in [ConfigFile.LOCAL, ConfigFile.ALL]:
-            ret.append(self._local.path)
+            ret.extend(self._local.paths)
         return ret
 
     def get(
@@ -282,7 +293,6 @@ def set(self, option: str, value: Any, configfile: ConfigFile = ConfigFile.LOCAL
         :param value: value to set option to
         :param configfile: type of config file to set the value in
         '''
-
         if configfile == ConfigFile.ALL:
             # We need a real configuration file; ALL doesn't make sense here.
             raise ValueError(configfile)

src/west/util.py

@@ -53,6 +53,10 @@ class WestNotFound(RuntimeError):
     '''Neither the current directory nor any parent has a west workspace.'''
 
 
+def paths_to_str(paths: list[pathlib.Path], sep: str = os.pathsep) -> str:
+    return sep.join([str(p) for p in paths])
+
+
 def west_dir(start: PathType | None = None) -> str:
     '''Returns the absolute path of the workspace's .west directory.
 

tests/test_config.py

@@ -10,7 +10,7 @@
 from typing import Any
 
 import pytest
-from conftest import cmd, cmd_raises, update_env
+from conftest import WINDOWS, cmd, cmd_raises, update_env
 
 from west import configuration as config
 from west.util import PathType
@@ -590,6 +590,91 @@ def test_config_precedence():
     assert cfg(f=LOCAL)['pytest']['precedence'] == 'local'
 
 
+def test_config_multiple(config_tmpdir):
+    # Verify that local settings take precedence over global ones,
+    # but that both values are still available, and that setting
+    # either doesn't affect system settings.
+    def write_config(config_file, section, key1, value1, key2, value2):
+        config_file.parent.mkdir(exist_ok=True)
+
+        content = textwrap.dedent(f'''
+        [{section}]
+        {key1} = {value1}
+        {key2} = {value2}
+        ''')
+
+        with open(config_file, 'w') as conf:
+            conf.write(content)
+
+    # config file paths
+    config_dir = pathlib.Path(config_tmpdir) / 'configs'
+    config_s1 = config_dir / 'system 1'
+    config_s2 = config_dir / 'system 2'
+    config_g1 = config_dir / 'global 1'
+    config_g2 = config_dir / 'global 2'
+    config_l1 = config_dir / 'local 1'
+    config_l2 = config_dir / 'local 2'
+
+    # create some configs with
+    # - some individual option per config file
+    # - the same option defined in multiple configs
+    write_config(config_s1, 'sec', 's', '1 !"$&/()=?\\', 's1', '1 !"$&/()=?\\')
+    write_config(config_s2, 'sec', 's', '2', 's2', '2')
+    write_config(config_g1, 'sec', 'g', '1', 'g1', '1')
+    write_config(config_g2, 'sec', 'g', '2', 'g2', '2')
+    write_config(config_l1, 'sec', 'l', '1', 'l1', '1')
+    write_config(config_l2, 'sec', 'l', '2', 'l2', '2')
+
+    # use a non-readable config file (does not work on Windows)
+    if not WINDOWS:
+        config_non_readable = config_dir / 'non-readable'
+        config_non_readable.touch()
+        config_non_readable.chmod(0o000)
+        os.environ["WEST_CONFIG_GLOBAL"] = f'{config_g1}{os.pathsep}{config_non_readable}'
+        stderr = cmd_raises('config --global some.section', subprocess.CalledProcessError)
+        expected = f"Error while reading one of '{config_g1}{os.pathsep}{config_non_readable}'"
+        assert expected in stderr
+
+    # specify multiple configs for each config level (separated by os.pathsep)
+    os.environ["WEST_CONFIG_GLOBAL"] = f'{config_g1}{os.pathsep}{config_g2}'
+    os.environ["WEST_CONFIG_SYSTEM"] = f'{config_s1}{os.pathsep}{config_s2}'
+    os.environ["WEST_CONFIG_LOCAL"] = f'{config_l1}{os.pathsep}{config_l2}'
+
+    # check that all individual options are applied
+    stdout = cmd('config --system sec.s1').rstrip()
+    assert stdout == '1 !"$&/()=?\\'
+    stdout = cmd('config --system sec.s2').rstrip()
+    assert stdout == '2'
+    stdout = cmd('config --global sec.g1').rstrip()
+    assert stdout == '1'
+    stdout = cmd('config --global sec.g2').rstrip()
+    assert stdout == '2'
+    stdout = cmd('config --local sec.l1').rstrip()
+    assert stdout == '1'
+    stdout = cmd('config --local sec.l2').rstrip()
+    assert stdout == '2'
+
+    # check that options from latest config overrides
+    stdout = cmd('config --system sec.s').rstrip()
+    assert stdout == '2'
+    stdout = cmd('config --global sec.g').rstrip()
+    assert stdout == '2'
+    stdout = cmd('config --local sec.l').rstrip()
+    assert stdout == '2'
+
+    # check that list-paths gives correct output
+    stdout = cmd('config --global --list-paths')
+    assert [str(config_g1), str(config_g2)] == stdout.rstrip().splitlines()
+    stdout = cmd('config --system --list-paths')
+    assert [str(config_s1), str(config_s2)] == stdout.rstrip().splitlines()
+    stdout = cmd('config --local --list-paths')
+    assert [str(config_l1), str(config_l2)] == stdout.rstrip().splitlines()
+
+    # writing not possible if multiple configs are used
+    err_msg = cmd_raises('config --local sec.l3 3', subprocess.CalledProcessError)
+    assert f'Cannot write if multiple configs in use: {config_l1}{os.pathsep}{config_l2}' in err_msg
+
+
 def test_config_missing_key():
     err_msg = cmd_raises('config pytest', subprocess.CalledProcessError)
     assert 'invalid configuration option "pytest"; expected "section.key" format' in err_msg