fix: Fix filtering of files with default exclude filter patterns

Author: kumekay

docs/en/guides/packaging_components.rst

@@ -170,7 +170,7 @@ As a component developer, you may want to choose what files from the component d
      exclude:
        - "*.py"         # Exclude all Python files
        - "**/*.list"    # Exclude `.list` files in all directories
-       - "big_dir/**/*" # Exclude files in `big_dir` directory (but empty directory will be added to archive anyway)
+       - "big_dir/**/*" # Exclude `big_dir` directory and its content
      include:
        - "**/.DS_Store" # Include files excluded by default
 

docs/en/reference/manifest_file.rst

@@ -151,7 +151,7 @@ Example:
      exclude:
        - "*.py" # Exclude all Python files
        - "**/*.list" # Exclude `.list` files in all directories
-       - "big_dir/**/*" # Exclude files in `big_dir` directory (but the empty directory will be added to the archive anyway)
+       - "big_dir/**/*" # Exclude `big_dir` directory and its content
      include:
        - "**/.DS_Store" # Include files excluded by default
 

idf_component_tools/file_tools.py

@@ -79,13 +79,16 @@ def exclude_paths(pattern):
     if exclude_default:
         for pattern in DEFAULT_EXCLUDE:
             exclude_paths(pattern)
-            if pattern.endswith('/**/*'):
-                exclude_paths(pattern[: pattern.rindex('/**/*')])
 
     # Exclude user patterns
     for pattern in exclude:
         exclude_paths(pattern)
 
+    # Exclude all the directories
+    for path in list(paths):
+        if path.is_dir():
+            paths.remove(path)
+
     # Include everything that was explicitly added
     for pattern in include:
         include_paths(pattern)

tests/test_file_tools.py

@@ -22,46 +22,50 @@ def assets_path(tmp_path, fixtures_path):
     # Avoid `dirs_exist_ok=True` missing in python 2
     subdir = tmp_path / 'sub'
     shutil.copytree(templatepath.as_posix(), subdir.as_posix())
-    return subdir.as_posix()
+    return subdir
 
 
 def test_filtered_path_default(assets_path):
     assert filtered_paths(assets_path) == {
-        Path(assets_path, '1.txt'),
-        Path(assets_path, 'ignore.dir'),
-        Path(assets_path, 'ignore.dir', 'file.txt'),
-        Path(assets_path, 'ignore.me'),
+        assets_path / '1.txt',
+        assets_path / 'ignore.dir' / 'file.txt',
+        assets_path / 'ignore.me',
     }
 
 
 def test_filtered_path_no_default(assets_path):
     assert filtered_paths(assets_path, exclude_default=False) == {
-        Path(assets_path, '1.txt'),
-        Path(assets_path, 'ignore.dir'),
-        Path(assets_path, 'ignore.dir', 'file.txt'),
-        Path(assets_path, 'ignore.me'),
-        Path(assets_path, '.gitlab-ci.yml'),
+        assets_path / '1.txt',
+        assets_path / 'ignore.dir' / 'file.txt',
+        assets_path / 'ignore.me',
+        assets_path / '.gitlab-ci.yml',
     }
 
 
-def test_filtered_path_exclude_file(assets_path):
+def test_filtered_path_exclude_file_and_empty_dirs(assets_path):
     assert filtered_paths(assets_path, exclude=['**/file.txt']) == {
-        Path(assets_path, '1.txt'),
-        Path(assets_path, 'ignore.dir'),
-        Path(assets_path, 'ignore.me'),
+        assets_path / '1.txt',
+        assets_path / 'ignore.me',
     }
 
 
-def test_filtered_path_keep_empty_dir(assets_path):
+def test_filtered_path_exclude_file_and_empty_dir_kept(assets_path):
+    assert filtered_paths(assets_path, exclude=['**/file.txt'], include=['ignore.dir']) == {
+        assets_path / '1.txt',
+        assets_path / 'ignore.me',
+        assets_path / 'ignore.dir',
+    }
+
+
+def test_filtered_path_removes_empty_dir(assets_path):
     assert filtered_paths(
         assets_path,
         exclude=[
             'ignore.dir/**/*',
         ],
     ) == {
-        Path(assets_path, '1.txt'),
-        Path(assets_path, 'ignore.me'),
-        Path(assets_path, 'ignore.dir'),
+        assets_path / '1.txt',
+        assets_path / 'ignore.me',
     }
 
 
@@ -73,29 +77,38 @@ def test_filtered_path_exclude_empty_dir(assets_path):
             'ignore.dir/*',
         ],
     ) == {
-        Path(assets_path, '1.txt'),
-        Path(assets_path, 'ignore.me'),
+        assets_path / '1.txt',
+        assets_path / 'ignore.me',
     }
 
 
 def test_filtered_path_exclude_dir_with_file(assets_path):
-    extra_path = Path(assets_path, 'ignore.dir', 'extra').as_posix()
-    os.mkdir(extra_path)
-    one_more = os.path.join(extra_path, 'one_more.txt')
-    shutil.copy(os.path.join(assets_path, '1.txt'), one_more)
+    extra_path = assets_path / 'ignore.dir' / 'extra'
+    extra_path.mkdir(exist_ok=True)
+    one_more = extra_path / 'one_more.txt'
+    shutil.copy(assets_path / '1.txt', one_more)
 
-    assert os.path.exists(one_more)
+    assert one_more.exists()
 
     assert filtered_paths(
         assets_path,
         exclude=[
             'ignore.dir/*',
         ],
     ) == {
-        Path(assets_path, '1.txt'),
-        Path(assets_path, 'ignore.dir'),
-        Path(assets_path, 'ignore.dir', 'extra', 'one_more.txt'),
-        Path(assets_path, 'ignore.me'),
+        assets_path / '1.txt',
+        assets_path / 'ignore.dir' / 'extra' / 'one_more.txt',
+        assets_path / 'ignore.me',
+    }
+
+
+def test_filtered_with_default_path(tmp_path):
+    (tmp_path / 'build_all.sh').touch()
+    (tmp_path / 'build_me').mkdir()
+    (tmp_path / 'build_me' / 'file').touch()
+
+    assert filtered_paths(tmp_path, exclude_default=True) == {
+        tmp_path / 'build_all.sh',
     }