support multiple west config files per config level

multiple config files can be specified in the according environment
variable for each config level (separated by 'os.pathsep', which is ';'
on Windows or ':' otherwise). The config files are applied in the same
order as they are specified, whereby values from later files override
earlier ones.

Author: thorsten-klein

src/west/app/config.py

@@ -7,7 +7,7 @@
 import argparse
 
 from west.commands import CommandError, WestCommand
-from west.configuration import ConfigFile
+from west.configuration import ConfigFile, Configuration
 
 CONFIG_DESCRIPTION = '''\
 West configuration file handling.
@@ -35,13 +35,32 @@
 
 - Linux, macOS, Windows: <workspace-root-directory>/.west/config
 
-You can override these files' locations with the WEST_CONFIG_SYSTEM,
-WEST_CONFIG_GLOBAL, and WEST_CONFIG_LOCAL environment variables.
-
 Configuration values from later configuration files override configuration
 from earlier ones. Local values have highest precedence, and system values
 lowest.
 
+You can override the config file location for the according config level
+with the environment variables:
+- WEST_CONFIG_SYSTEM,
+- WEST_CONFIG_GLOBAL
+- WEST_CONFIG_LOCAL
+
+Note: west will NOT fail if a specified config file does not exist, but it
+will ignore this file.
+
+For each configuration level (local, global, and system) also multiple config
+file locations can be specified. To do so, set according environment variable
+to contain all paths (separated by 'os.pathsep', which is ';' on Windows or
+':' otherwise)
+
+You can list all existing configuration files that are currently considered
+(existing files) via the following command, which prints them in the exact
+order as they are loaded (so later values override earlier ones):
+    west config --list-paths
+    west config --local --list-paths
+    west config --global --list-paths
+    west config --system --list-paths
+
 To get a value for <name>, type:
     west config <name>
 
@@ -99,6 +118,11 @@ def do_add_parser(self, parser_adder):
             "action to perform (give at most one)"
         ).add_mutually_exclusive_group()
 
+        group.add_argument(
+            '--list-paths',
+            action='store_true',
+            help='list all config files that are currently considered by west config',
+        )
         group.add_argument(
             '-l', '--list', action='store_true', help='list all options and their values'
         )
@@ -153,13 +177,15 @@ def do_run(self, args, user_args):
         if args.list:
             if args.name:
                 self.parser.error('-l cannot be combined with name argument')
-        elif not args.name:
+        elif not args.name and not args.list_paths:
             self.parser.error('missing argument name (to list all options and values, use -l)')
         elif args.append:
             if args.value is None:
                 self.parser.error('-a requires both name and value')
 
-        if args.list:
+        if args.list_paths:
+            self.list_paths(args)
+        elif args.list:
             self.list(args)
         elif delete:
             self.delete(args)
@@ -170,6 +196,11 @@ def do_run(self, args, user_args):
         else:
             self.write(args)
 
+    def list_paths(self, args):
+        config_paths = Configuration().get_paths(args.configfile or ALL)
+        for config_path in config_paths:
+            print(config_path)
+
     def list(self, args):
         what = args.configfile or ALL
         for option, value in self.config.items(configfile=what):

src/west/configuration.py

@@ -68,19 +68,34 @@ def parse_key(dotted_name: str):
         return section_child
 
     @staticmethod
-    def from_path(path: Path | None) -> '_InternalCF | None':
-        return _InternalCF(path) if path and path.exists() else None
-
-    def __init__(self, path: Path):
-        self.path = path
+    def from_path(path: Path | list[Path] | None) -> '_InternalCF | None':
+        if not path:
+            return None
+
+        # Normalize to a list of existing paths
+        paths = path
+        if isinstance(paths, Path):
+            paths = [Path(p) for p in str(paths).split(os.pathsep)]
+        paths = [p for p in paths if p.exists()]
+        return _InternalCF(paths) if paths else None
+
+    def __init__(self, paths: list[Path]):
         self.cp = _configparser()
-        read_files = self.cp.read(path, encoding='utf-8')
-        if len(read_files) != 1:
-            raise FileNotFoundError(path)
+        self.paths = paths
+        read_files = self.cp.read(self.paths, encoding='utf-8')
+        if len(read_files) != len(self.paths):
+            raise FileNotFoundError(paths)
+
+    def _write(self):
+        if not self.paths:
+            raise WestNotFound('No config file exists that can be written')
+        if len(self.paths) > 1:
+            raise WestNotFound('Cannot write if multiple configs in use.')
+        with open(self.paths[0], 'w', encoding='utf-8') as f:
+            self.cp.write(f)
 
     def __contains__(self, option: str) -> bool:
         section, key = _InternalCF.parse_key(option)
-
         return section in self.cp and key in self.cp[section]
 
     def get(self, option: str):
@@ -95,37 +110,31 @@ def getint(self, option: str):
     def getfloat(self, option: str):
         return self._get(option, self.cp.getfloat)
 
-    def _get(self, option, getter):
+    def _get(self, option, config_getter):
         section, key = _InternalCF.parse_key(option)
-
-        try:
-            return getter(section, key)
-        except (configparser.NoOptionError, configparser.NoSectionError) as err:
-            raise KeyError(option) from err
+        if section in self.cp and key in self.cp[section]:
+            getter = config_getter
+        else:
+            raise KeyError(option)
+        return getter(section, key)
 
     def set(self, option: str, value: Any):
         section, key = _InternalCF.parse_key(option)
-
         if section not in self.cp:
             self.cp[section] = {}
-
         self.cp[section][key] = value
-
-        with open(self.path, 'w', encoding='utf-8') as f:
-            self.cp.write(f)
+        self._write()
 
     def delete(self, option: str):
         section, key = _InternalCF.parse_key(option)
-
-        if section not in self.cp:
+        if option not in self:
             raise KeyError(option)
 
         del self.cp[section][key]
         if not self.cp[section].items():
             del self.cp[section]
 
-        with open(self.path, 'w', encoding='utf-8') as f:
-            self.cp.write(f)
+        self._write()
 
 
 class ConfigFile(Enum):
@@ -181,6 +190,16 @@ def __init__(self, topdir: PathType | None = None):
         self._global = _InternalCF.from_path(self._global_path)
         self._local = _InternalCF.from_path(self._local_path)
 
+    def get_paths(self, configfile: ConfigFile = ConfigFile.ALL):
+        ret = []
+        if self._global and configfile in [ConfigFile.GLOBAL, ConfigFile.ALL]:
+            ret += self._global.paths
+        if self._system and configfile in [ConfigFile.SYSTEM, ConfigFile.ALL]:
+            ret += self._system.paths
+        if self._local and configfile in [ConfigFile.LOCAL, ConfigFile.ALL]:
+            ret += self._local.paths
+        return ret
+
     def get(
         self, option: str, default: str | None = None, configfile: ConfigFile = ConfigFile.ALL
     ) -> str | None:
@@ -269,7 +288,9 @@ def set(self, option: str, value: Any, configfile: ConfigFile = ConfigFile.LOCAL
         :param value: value to set option to
         :param configfile: type of config file to set the value in
         '''
-
+        paths = self.get_paths(configfile)
+        if len(paths) > 1:
+            raise WestNotFound('Cannot write if multiple configs in use.')
         if configfile == ConfigFile.ALL:
             # We need a real configuration file; ALL doesn't make sense here.
             raise ValueError(configfile)

tests/test_config.py

@@ -6,6 +6,7 @@
 import os
 import pathlib
 import subprocess
+import textwrap
 from typing import Any
 
 import pytest
@@ -93,6 +94,62 @@ def test_config_global():
     assert 'pytest' not in lcl
 
 
+TEST_CASES_CONFIG_LIST_PATHS = [
+    # (flag, env_var)
+    ('--local', 'WEST_CONFIG_LOCAL'),
+    ('--system', 'WEST_CONFIG_SYSTEM'),
+    ('--global', 'WEST_CONFIG_GLOBAL'),
+]
+
+
+@pytest.mark.parametrize("test_case", TEST_CASES_CONFIG_LIST_PATHS)
+def test_config_list_paths(test_case):
+    flag, env_var = test_case
+
+    # create the config
+    cmd(f'config {flag} pytest.key val')
+
+    # check that the config is listed now
+    stdout = cmd(f'config {flag} --list-paths')
+    config_path = pathlib.Path(os.environ[env_var])
+    assert f'{config_path}' == stdout.rstrip()
+
+    # no config is listed (since it does not exist)
+    config_path.unlink()
+    stdout = cmd(f'config {flag} --list-paths')
+    assert '' == stdout.rstrip()
+
+
+def test_config_list_paths_extended():
+    WEST_CONFIG_LOCAL = os.environ['WEST_CONFIG_LOCAL']
+    WEST_CONFIG_GLOBAL = os.environ['WEST_CONFIG_GLOBAL']
+    WEST_CONFIG_SYSTEM = os.environ['WEST_CONFIG_SYSTEM']
+
+    # create the configs
+    cmd('config --local pytest.key val')
+    cmd('config --global pytest.key val')
+    cmd('config --system pytest.key val')
+
+    # list the configs
+    stdout = cmd('config --list-paths')
+    assert (
+        stdout.splitlines()
+        == textwrap.dedent(f'''\
+        {WEST_CONFIG_GLOBAL}
+        {WEST_CONFIG_SYSTEM}
+        {WEST_CONFIG_LOCAL}
+        ''').splitlines()
+    )
+
+    # do not list any configs if no config files exist
+    # (Note: even no local config exists, same as outside any west workspace)
+    pathlib.Path(WEST_CONFIG_GLOBAL).unlink()
+    pathlib.Path(WEST_CONFIG_SYSTEM).unlink()
+    pathlib.Path(WEST_CONFIG_LOCAL).unlink()
+    stdout = cmd('config --list-paths')
+    assert stdout.splitlines() == []
+
+
 def test_config_local():
     # test_config_system for local variables.
     cmd('config --local pytest.local foo')
@@ -519,6 +576,81 @@ def test_config_precedence():
     assert cfg(f=LOCAL)['pytest']['precedence'] == 'local'
 
 
+def test_config_multiple(config_tmpdir):
+    # Verify that local settings take precedence over global ones,
+    # but that both values are still available, and that setting
+    # either doesn't affect system settings.
+    def write_config(config_file, section, key1, value1, key2, value2):
+        config_file.parent.mkdir(exist_ok=True)
+
+        content = textwrap.dedent(f'''
+        [{section}]
+        {key1} = {value1}
+        {key2} = {value2}
+        ''')
+
+        with open(config_file, 'w') as conf:
+            conf.write(content)
+
+    # config file paths
+    config_dir = pathlib.Path(config_tmpdir) / 'configs'
+    config_s1 = config_dir / 'system1'
+    config_s2 = config_dir / 'system2'
+    config_g1 = config_dir / 'global1'
+    config_g2 = config_dir / 'global2'
+    config_l1 = config_dir / 'local1'
+    config_l2 = config_dir / 'local2'
+
+    # create some configs with
+    # - some individual option per config file
+    # - the same option defined in multiple configs
+    write_config(config_s1, 'sec', 's', 1, 's1', 1)
+    write_config(config_s2, 'sec', 's', 2, 's2', 2)
+    write_config(config_g1, 'sec', 'g', 1, 'g1', 1)
+    write_config(config_g2, 'sec', 'g', 2, 'g2', 2)
+    write_config(config_l1, 'sec', 'l', 1, 'l1', 1)
+    write_config(config_l2, 'sec', 'l', 2, 'l2', 2)
+
+    # specify multiple configs for each config level (separated by os.pathsep)
+    os.environ["WEST_CONFIG_GLOBAL"] = f'{config_g1}{os.pathsep}{config_g2}'
+    os.environ["WEST_CONFIG_SYSTEM"] = f'{config_s1}{os.pathsep}{config_s2}'
+    os.environ["WEST_CONFIG_LOCAL"] = f'{config_l1}{os.pathsep}{config_l2}'
+
+    # check that all individual options are applied
+    stdout = cmd('config --system sec.s1').rstrip()
+    assert stdout == '1'
+    stdout = cmd('config --system sec.s2').rstrip()
+    assert stdout == '2'
+    stdout = cmd('config --global sec.g1').rstrip()
+    assert stdout == '1'
+    stdout = cmd('config --global sec.g2').rstrip()
+    assert stdout == '2'
+    stdout = cmd('config --local sec.l1').rstrip()
+    assert stdout == '1'
+    stdout = cmd('config --local sec.l2').rstrip()
+    assert stdout == '2'
+
+    # check that options from latest config overrides
+    stdout = cmd('config --system sec.s').rstrip()
+    assert stdout == '2'
+    stdout = cmd('config --global sec.g').rstrip()
+    assert stdout == '2'
+    stdout = cmd('config --local sec.l').rstrip()
+    assert stdout == '2'
+
+    # check that list-paths gives correct output
+    stdout = cmd('config --global --list-paths')
+    assert [str(config_g1), str(config_g2)] == stdout.rstrip().splitlines()
+    stdout = cmd('config --system --list-paths')
+    assert [str(config_s1), str(config_s2)] == stdout.rstrip().splitlines()
+    stdout = cmd('config --local --list-paths')
+    assert [str(config_l1), str(config_l2)] == stdout.rstrip().splitlines()
+
+    # writing not possible if multiple configs are used
+    err_msg = cmd_raises('config --local sec.l3 3', subprocess.CalledProcessError)
+    assert 'Cannot write if multiple configs in use' in err_msg
+
+
 def test_config_missing_key():
     err_msg = cmd_raises('config pytest', subprocess.CalledProcessError)
     assert 'invalid configuration option "pytest"; expected "section.key" format' in err_msg