support config dropin directories

config files in acccording dropin directories are considered
automatically. The dropin directory is named as the config itself but
with a '.d' suffix. For example for local config the dropin directory is
'./west/config.d' and for global config it is '~/.westconfig.d'

support config dropin directories

config files in acccording dropin directories are considered
automatically. The dropin directory is named as the config itself but
with a '.d' suffix. For example for local config the dropin directory is
'./west/config.d' and for global config it is '~/.westconfig.d'

Boolean option `config.dropins` must be set to `true` so that dropin
configs for the according config type are considered. So if the option
config.dropins=true in enabled in local config, only local dropin
configs are used (but no system or global ones).

Author: thorsten-klein

src/west/app/config.py

@@ -82,6 +82,20 @@
 To delete <name> everywhere it's set, including the system file:
     west config -D <name>
 
+For each configuration type (local, global, and system), an additional
+drop-in config directory is supported. This directory is named as the
+according config file, but with a '.d' suffix, whereby all '.conf' and
+'.ini' files are loaded in alphabetical order.
+
+All files inside a drop-in directory must use `.conf` extension and are
+loaded in **alphabetical order**.
+For example:
+    .west/config.d/basics.conf
+
+Note: It is not possible to modify dropin configs.via 'west config' commands.
+When config option values are set/appended/deleted via 'west config' commands,
+always the config file is modified (never the dropin config files).
+
 To list the configuration files that are loaded (both the main config file
 and all drop-ins) in the exact order they were applied (where later values
 override earlier ones):

src/west/configuration.py

@@ -59,6 +59,11 @@ class _InternalCF:
     # For internal use only; convenience interface for reading and
     # writing INI-style [section] key = value configuration files,
     # but presenting a west-style section.key = value style API.
+    # The config file and the drop-in configs need separate
+    # configparsers, since west needs to determine options that are
+    # set in the config. E.g. if config values are updated, only those
+    # values shall be written to the config, which are already present
+    # (and not all values that may also come from dropin configs)
 
     @staticmethod
     def parse_key(dotted_name: str):
@@ -69,63 +74,91 @@ def parse_key(dotted_name: str):
 
     @staticmethod
     def from_path(path: Path | None) -> '_InternalCF | None':
-        return _InternalCF(path) if path and path.exists() else None
+        if not path:
+            return None
+        cf = _InternalCF(path)
+        if not cf.path and not cf.dropin_paths:
+            return None
+        return cf
 
     def __init__(self, path: Path):
-        self.path = path
         self.cp = _configparser()
-        read_files = self.cp.read(path, encoding='utf-8')
-        if len(read_files) != 1:
-            raise FileNotFoundError(path)
+        self.path = path if path.exists() else None
+        if self.path:
+            self.cp.read(self.path, encoding='utf-8')
+
+        # consider dropin configs
+        self.dropin_cp = _configparser()
+        self.dropin_dir = None
+        self.dropin_paths = []
+        # dropin configs must be enabled in config
+        if self.cp.getboolean('config', 'dropins', fallback=False):
+            # dropin dir is the config path with .d suffix
+            dropin_dir = Path(f'{path}.d')
+            self.dropin_dir = dropin_dir if dropin_dir.exists() else None
+            if self.dropin_dir:
+                # dropin configs are applied in alphabetical order
+                for conf in sorted(self.dropin_dir.iterdir()):
+                    # only consider .conf files
+                    if conf.suffix in ['.conf', '.ini']:
+                        self.dropin_paths.append(self.dropin_dir / conf)
+                if self.dropin_paths:
+                    self.dropin_cp.read(self.dropin_paths, encoding='utf-8')
+
+    def _paths(self) -> list[Path]:
+        ret = [p for p in self.dropin_paths]
+        if self.path:
+            ret.append(self.path)
+        return ret
+
+    def _write(self):
+        with open(self.path, 'w', encoding='utf-8') as f:
+            self.cp.write(f)
 
     def __contains__(self, option: str) -> bool:
         section, key = _InternalCF.parse_key(option)
 
-        return section in self.cp and key in self.cp[section]
+        if section in self.cp and key in self.cp[section]:
+            return True
+        return section in self.dropin_cp and key in self.dropin_cp[section]
 
     def get(self, option: str):
-        return self._get(option, self.cp.get)
+        return self._get(option, self.cp.get, self.dropin_cp.get)
 
     def getboolean(self, option: str):
-        return self._get(option, self.cp.getboolean)
+        return self._get(option, self.cp.getboolean, self.dropin_cp.getboolean)
 
     def getint(self, option: str):
-        return self._get(option, self.cp.getint)
+        return self._get(option, self.cp.getint, self.dropin_cp.getint)
 
     def getfloat(self, option: str):
-        return self._get(option, self.cp.getfloat)
+        return self._get(option, self.cp.getfloat, self.dropin_cp.getfloat)
 
-    def _get(self, option, getter):
+    def _get(self, option, config_getter, dropin_getter):
         section, key = _InternalCF.parse_key(option)
-
-        try:
-            return getter(section, key)
-        except (configparser.NoOptionError, configparser.NoSectionError) as err:
-            raise KeyError(option) from err
+        if section in self.cp and key in self.cp[section]:
+            getter = config_getter
+        elif section in self.dropin_cp and key in self.dropin_cp[section]:
+            getter = dropin_getter
+        else:
+            raise KeyError(option)
+        return getter(section, key)
 
     def set(self, option: str, value: Any):
         section, key = _InternalCF.parse_key(option)
-
         if section not in self.cp:
             self.cp[section] = {}
-
         self.cp[section][key] = value
-
-        with open(self.path, 'w', encoding='utf-8') as f:
-            self.cp.write(f)
+        self._write()
 
     def delete(self, option: str):
         section, key = _InternalCF.parse_key(option)
-
-        if section not in self.cp:
+        if option not in self:
             raise KeyError(option)
-
         del self.cp[section][key]
         if not self.cp[section].items():
             del self.cp[section]
-
-        with open(self.path, 'w', encoding='utf-8') as f:
-            self.cp.write(f)
+        self._write()
 
 
 class ConfigFile(Enum):
@@ -183,22 +216,22 @@ def __init__(self, topdir: PathType | None = None):
 
     def get_path(self, configfile: ConfigFile = ConfigFile.LOCAL):
         if configfile == ConfigFile.ALL:
-            raise RuntimeError(f'{configfile} not allowed for get_path')
+            raise RuntimeError(f'{configfile} not allowed for this operation')
         elif configfile == ConfigFile.LOCAL:
             if not self._local_path:
                 raise MalformedConfig('local configuration cannot be determined')
             return self._local_path
-        elif configfile == ConfigFile.SYSTEM:
-            return self._system_path
         elif configfile == ConfigFile.GLOBAL:
             return self._global_path
+        elif configfile == ConfigFile.SYSTEM:
+            return self._system_path
 
     def get_paths(self, configfile: ConfigFile = ConfigFile.ALL):
         ret = []
-        if self._global and configfile in [ConfigFile.GLOBAL, ConfigFile.ALL]:
-            ret += self._global._paths()
         if self._system and configfile in [ConfigFile.SYSTEM, ConfigFile.ALL]:
             ret += self._system._paths()
+        if self._global and configfile in [ConfigFile.GLOBAL, ConfigFile.ALL]:
+            ret += self._global._paths()
         if self._local and configfile in [ConfigFile.LOCAL, ConfigFile.ALL]:
             ret += self._local._paths()
         return ret
@@ -376,13 +409,14 @@ def _copy_to_configparser(self, cp: configparser.ConfigParser) -> None:
         # function-and-global-state APIs.
 
         def load(cf: _InternalCF):
-            for section, contents in cf.cp.items():
-                if section == 'DEFAULT':
-                    continue
-                if section not in cp:
-                    cp.add_section(section)
-                for key, value in contents.items():
-                    cp[section][key] = value
+            for cp in [cf.dropin_cp, cf.cp]:
+                for section, contents in cp.items():
+                    if section == 'DEFAULT':
+                        continue
+                    if section not in cp:
+                        cp.add_section(section)
+                    for key, value in contents.items():
+                        cp[section][key] = value
 
         if self._system:
             load(self._system)
@@ -428,11 +462,12 @@ def _cf_to_dict(cf: _InternalCF | None) -> dict[str, Any]:
         ret: dict[str, Any] = {}
         if cf is None:
             return ret
-        for section, contents in cf.cp.items():
-            if section == 'DEFAULT':
-                continue
-            for key, value in contents.items():
-                ret[f'{section}.{key}'] = value
+        for cp in [cf.dropin_cp, cf.cp]:
+            for section, contents in cp.items():
+                if section == 'DEFAULT':
+                    continue
+                for key, value in contents.items():
+                    ret[f'{section}.{key}'] = value
         return ret