Merge pull request #266 from espressif/feat/add_support_of_negation_config_rules

feat: add support of negation config rules

Author: hfudev

docs/en/explanations/config_rules.rst

@@ -52,6 +52,11 @@ To define a Config Rule, use the following format: ``[SDKCONFIG_FILEPATTERN]=[CO
 -  ``SDKCONFIG_FILEPATTERN``: This can be a file name to match a `sdkconfig file <#sdkconfig-files>`_ or a pattern with one wildcard (``*``) character to match multiple `sdkconfig files`_.
 -  ``CONFIG_NAME``: The name of the corresponding build configuration. This value can be skipped if the wildcard value is to be used.
 
+To exclude specific `sdkconfig files`_, use the negation rule format: ``![SDKCONFIG_FILEPATTERN]``.
+
+-  ``SDKCONFIG_FILEPATTERN``: The format is the same as in the normal Config Rule.
+-  Negation rules are applied after all inclusion rules, so the order of negation rules does not matter.
+
 The config rules and the corresponding matched `sdkconfig files`_ for the example project are as follows:
 
 .. list-table:: Config Rules
@@ -85,6 +90,30 @@ The config rules and the corresponding matched `sdkconfig files`_ for the exampl
       -  -  ``sdkconfig.ci.foo``
          -  ``sdkconfig.ci.bar``
 
+   -  -  -  ``sdkconfig.ci.*=``
+         -  ``!sdkconfig.ci.test``
+      -  -  ``foo``
+         -  ``bar``
+      -  The negation rule excludes the ``sdkconfig.ci.test`` file.
+      -  -  ``sdkconfig.ci.foo``
+         -  ``sdkconfig.ci.bar``
+
+   -  -  -  ``sdkconfig.ci.*=``
+         -  ``!sdkconfig.ci.test*``
+      -  -  ``foo``
+         -  ``bar``
+      -  The negation rule excludes the files matching the negation wildcard pattern like ``sdkconfig.ci.test`` or ``sdkconfig.ci.test_debug``.
+      -  -  ``sdkconfig.ci.foo``
+         -  ``sdkconfig.ci.bar``
+
+   -  -  -  ``!sdkconfig.ci.test*``
+         -  ``sdkconfig.ci.*=``
+      -  -  ``foo``
+         -  ``bar``
+      -  Same as the previous example, but the negation rule is applied first.
+      -  -  ``sdkconfig.ci.foo``
+         -  ``sdkconfig.ci.bar``
+
 ****************
  Override Order
 ****************

docs/en/explanations/find.rst

@@ -89,6 +89,14 @@ The output would be:
    (cmake) App test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.ci.foo, build in test-1/build
    (cmake) App test-1, target esp32, sdkconfig /tmp/test/examples/test-1/sdkconfig.defaults, build in test-1/build
 
+To drop specific sdkconfigs, add a negation rule:
+
+.. code:: shell
+
+   idf-build-apps find -p test-1 --target esp32 --config "sdkconfig.ci.*=" "!sdkconfig.ci.bar"
+
+The ``bar`` configuration is omitted, only ``foo`` remains.
+
 .. _find-placeholders:
 
 ****************************

docs/en/references/config_file.rst

@@ -42,6 +42,7 @@ Here's a simple example of a configuration file:
          # config rules
          config = [
              "sdkconfig.*=",
+             "!sdkconfig.test",
              "=default",
          ]
 
@@ -62,6 +63,7 @@ Here's a simple example of a configuration file:
          # config rules
          config = [
              "sdkconfig.*=",
+             "!sdkconfig.test",
              "=default",
          ]
 
@@ -73,7 +75,7 @@ Running ``idf-build-apps build`` with the above configuration is equivalent to t
      --paths components examples \
      --target esp32 \
      --recursive \
-     --config-rules "sdkconfig.*=" "=default" \
+     --config-rules "sdkconfig.*=" "!sdkconfig.test" "=default" \
      --build-dir "build_@t_@w"
 
 `TOML <https://toml.io/en/>`__ supports native data types. In order to get the config name and type of the corresponding CLI option, you may refer to the help messages by using ``idf-build-apps find -h`` or ``idf-build-apps build -h``.

idf_build_apps/args.py

@@ -565,7 +565,9 @@ class FindBuildArguments(DependencyDrivenBuildArguments):
         'Optional NAME is the name of the configuration. '
         'if not specified, the filename is used as the name. '
         'FILEPATTERN is the filename of the sdkconfig file with a single wildcard character (*). '
-        'The NAME is the value matched by the wildcard',
+        'The NAME is the value matched by the wildcard. '
+        'Prefix a rule with ! to exclude matching files from the results (e.g. !sdkconfig.ci.test). '
+        'Negation rules are applied globally after all positive rules',
         validation_alias=AliasChoices('config_rules', 'config_rules_str', 'config'),
         default=None,
     )

idf_build_apps/finder.py

@@ -46,6 +46,9 @@ def _get_apps_from_path(
     sdkconfig_paths_matched = False
 
     for rule in config_rules:
+        if rule.negated:
+            continue
+
         if not rule.file_name:
             default_config_name = rule.config_name
             continue
@@ -71,6 +74,17 @@ def _get_apps_from_path(
 
             app_configs.append((sdkconfig_path, config_name))
 
+    # Apply negation rules
+    negated_paths: t.Set[str] = set()
+    for rule in config_rules:
+        if not rule.negated:
+            continue
+        for matched in Path(path).glob(rule.file_name):
+            negated_paths.add(str(matched.resolve()))
+
+    if negated_paths:
+        app_configs = [(p, n) for p, n in app_configs if p not in negated_paths]
+
     # no config rules matched, use default app
     if not sdkconfig_paths_matched:
         app_configs.append((None, default_config_name))

idf_build_apps/utils.py

@@ -20,30 +20,40 @@
 
 
 class ConfigRule:
-    def __init__(self, file_name: str, config_name: str = '') -> None:
+    def __init__(self, file_name: str, config_name: str = '', negated: bool = False) -> None:
         """
         ConfigRule represents the sdkconfig file and the config name.
 
         For example:
 
             - filename='', config_name='default' - represents the default app configuration, and gives it a name
                 'default'
-            - filename='sdkconfig.*', config_name=None - represents the set of configurations, names match the wildcard
+            - filename='sdkconfig.*', config_name='' - represents the set of configurations, names match the wildcard
                 value
+            - filename='sdkconfig.ci.test', negated=True - represents an exclusion rule, files matching this pattern
+                will be excluded from the build configurations
 
         :param file_name: name of the sdkconfig file fragment, optionally with a single wildcard ('*' character).
             can also be empty to indicate that the default configuration of the app should be used
-        :param config_name: name of the corresponding build configuration, or None if the value of wildcard is to be
-            used
+        :param config_name: name of the corresponding build configuration, or empty string if the value of the
+            wildcard is to be used
+        :param negated: if True, this rule excludes matching files instead of including them
         """
         self.file_name = file_name
         self.config_name = config_name
+        self.negated = negated
 
 
 def config_rules_from_str(rule_strings: t.Optional[t.List[str]]) -> t.List[ConfigRule]:
     """
     Helper function to convert strings like 'file_name=config_name' into `ConfigRule` objects
 
+    Supports the following formats:
+
+    - ``file_name=config_name`` - include the sdkconfig file with the given config name
+    - ``file_pattern=`` - include all sdkconfig files matching the pattern, config name is derived from the wildcard
+    - ``!file_name`` or ``!file_pattern`` - exclude matching sdkconfig files from the results
+
     :param rule_strings: list of rules as strings or a single rule string
     :return: list of ConfigRules
     """
@@ -52,10 +62,20 @@ def config_rules_from_str(rule_strings: t.Optional[t.List[str]]) -> t.List[Confi
 
     rules = []
     for rule_str in to_list(rule_strings):
+        if rule_str.startswith('!'):
+            negated_str = rule_str[1:].strip()
+            if '=' in negated_str:
+                raise InvalidInput(f'Negation rules must not have a config name: {rule_str}')
+            if not negated_str:
+                raise InvalidInput(f'Negation rules must contain a non-empty file name or pattern: {rule_str}')
+            rules.append(ConfigRule(negated_str, negated=True))
+            continue
+
         items = rule_str.split('=', 2)
         rules.append(ConfigRule(items[0], items[1] if len(items) == 2 else ''))
     # '' is the default config, sort this one to the front
-    return sorted(rules, key=lambda x: x.file_name)
+    # negated rules are sorted to the end since they are applied after positive rules
+    return sorted(rules, key=lambda x: (x.negated, x.file_name))
 
 
 def get_parallel_start_stop(total: int, parallel_count: int, parallel_index: int) -> t.Tuple[int, int]:

tests/test_finder.py

@@ -723,6 +723,90 @@ def test_env_var(self, tmp_path, monkeypatch):
         monkeypatch.delenv('TEST_ENV_VAR')
 
 
+@pytest.mark.parametrize(
+    'sdkconfig_files, config_rules, expected_config_names',
+    [
+        # basic negation
+        (
+            ['sdkconfig.ci.foo', 'sdkconfig.ci.bar', 'sdkconfig.ci.test'],
+            ['sdkconfig.ci.*=', '!sdkconfig.ci.test'],
+            ['bar', 'foo'],
+        ),
+        # negation with wildcard
+        (
+            ['sdkconfig.ci.foo', 'sdkconfig.ci.test_debug', 'sdkconfig.ci.test_release'],
+            ['sdkconfig.ci.*=', '!sdkconfig.ci.test*'],
+            ['foo'],
+        ),
+        # multiple negations
+        (
+            ['sdkconfig.ci.foo', 'sdkconfig.ci.bar', 'sdkconfig.ci.test', 'sdkconfig.ci.debug'],
+            ['sdkconfig.ci.*=', '!sdkconfig.ci.test', '!sdkconfig.ci.debug'],
+            ['bar', 'foo'],
+        ),
+        # negation-only (no positive match) -> default config
+        (
+            ['sdkconfig.ci.test'],
+            ['!sdkconfig.ci.test'],
+            [''],
+        ),
+        # negation with named config
+        (
+            ['sdkconfig.ci', 'sdkconfig.ci.foo', 'sdkconfig.ci.bar', 'sdkconfig.ci.test'],
+            ['sdkconfig.ci.*=', 'sdkconfig.ci=default', '!sdkconfig.ci.test'],
+            ['bar', 'default', 'foo'],
+        ),
+        # order independent (negation before positive rule)
+        (
+            ['sdkconfig.ci.foo', 'sdkconfig.ci.test'],
+            ['!sdkconfig.ci.test', 'sdkconfig.ci.*='],
+            ['foo'],
+        ),
+        # negation of non-existent file has no effect
+        (
+            ['sdkconfig.ci.foo', 'sdkconfig.ci.bar'],
+            ['sdkconfig.ci.*=', '!sdkconfig.ci.nonexistent'],
+            ['bar', 'foo'],
+        ),
+        # negation with whitespaces
+        (
+            ['sdkconfig.ci.foo', 'sdkconfig.ci.test'],
+            ['sdkconfig.ci.*=', '!   sdkconfig.ci.test   '],
+            ['foo'],
+        ),
+    ],
+)
+def test_config_rules_negation(tmp_path, sdkconfig_files, config_rules, expected_config_names):
+    create_project('test1', tmp_path)
+    for f in sdkconfig_files:
+        (tmp_path / 'test1' / f).touch()
+
+    apps = find_apps(
+        str(tmp_path / 'test1'),
+        'esp32',
+        recursive=True,
+        config_rules_str=config_rules,
+    )
+    assert len(apps) == len(expected_config_names)
+    assert sorted([app.config_name for app in apps]) == sorted(expected_config_names)
+
+
+def test_config_rules_negation_invalid_format():
+    from idf_build_apps.utils import InvalidInput
+    from idf_build_apps.utils import config_rules_from_str
+
+    with pytest.raises(InvalidInput, match='Negation rules must not have a config name'):
+        config_rules_from_str(['!sdkconfig.ci.test=myname'])
+
+
+def test_config_rules_negation_empty_pattern():
+    from idf_build_apps.utils import InvalidInput
+    from idf_build_apps.utils import config_rules_from_str
+
+    with pytest.raises(InvalidInput, match='Negation rules must contain a non-empty file name or pattern'):
+        config_rules_from_str(['!'])
+
+
 @pytest.mark.parametrize(
     'exclude_list, apps_count',
     [