Merge branch 'feat/upload_examples_defined_in_manifest_separately' into 'main'

feat: Upload examples defined in manifest separately

See merge request espressif/idf-component-manager!491

Author: kumekay

idf_component_manager/core.py

@@ -5,6 +5,7 @@
 import functools
 import os
 import re
+import secrets
 import shutil
 import tarfile
 import tempfile
@@ -78,7 +79,7 @@
     ProgressBar,
     _create_manifest_if_missing,
     archive_filename,
-    copy_examples_folders,
+    check_examples_folder,
     dist_name,
     get_validated_manifest,
     parse_example,
@@ -358,7 +359,7 @@ def pack_component(
         repository: t.Optional[str] = None,
         commit_sha: t.Optional[str] = None,
         repository_path: t.Optional[str] = None,
-    ) -> t.Tuple[str, Manifest]:
+    ) -> t.Tuple[str, t.Optional[str], Manifest]:
         dest_path = self.path / dest_dir if dest_dir else self.default_dist_path
 
         if version == 'git':
@@ -401,24 +402,43 @@ def pack_component(
             exclude=exclude_set,
         )
 
-        if manifest.examples:
-            copy_examples_folders(
-                manifest.examples,
-                Path(self.path),
-                dest_temp_dir,
-                use_gitignore=manifest.use_gitignore,
-                include=manifest.include_set,
-                exclude=manifest.exclude_set,
-            )
-
         manifest_manager.dump(str(dest_temp_dir))
 
         get_validated_manifest(manifest_manager, str(dest_temp_dir))
 
         archive_filepath = os.path.join(dest_path, archive_filename(name, manifest.version))
-        notice(f'Saving archive to "{archive_filepath}"')
+        notice(f'Saving component archive to "{archive_filepath}"')
         pack_archive(str(dest_temp_dir), archive_filepath)
-        return archive_filepath, manifest
+
+        if not manifest.examples:
+            return archive_filepath, None, manifest
+
+        check_examples_folder(manifest.examples, Path(self.path))
+
+        # Create a destination directory for examples defined in the manifest
+        examples_dest_dir = dest_path / f'{name}_{manifest.version}_examples'
+        examples_dest_dir.mkdir(parents=True, exist_ok=True)
+
+        for example in manifest.examples:
+            example_path = (self.path / Path(list(example.values())[0])).resolve()
+            # Do not consider examples from the `examples` directory
+            if Path(os.path.relpath(example_path, self.path)).parts[0] == 'examples':
+                continue
+
+            # Create a random directory to avoid conflicts with other examples
+            copy_filtered_directory(
+                example_path.as_posix(),
+                (examples_dest_dir / secrets.token_hex(4) / example_path.name).as_posix(),
+                use_gitignore=manifest.use_gitignore,
+                include=manifest.include_set,
+                exclude=exclude_set,
+            )
+
+        examples_archive_filepath = f'{examples_dest_dir}.tgz'
+        pack_archive(examples_dest_dir.as_posix(), examples_archive_filepath)
+        notice(f'Saving examples archive to "{examples_archive_filepath}"')
+
+        return archive_filepath, examples_archive_filepath, manifest
 
     @general_error_handler
     def delete_version(
@@ -518,6 +538,8 @@ def upload_component(
         """
         api_client = get_api_client(namespace=namespace, profile_name=profile_name)
 
+        examples_archive = []
+
         if archive:
             if version:
                 raise FatalError(
@@ -539,7 +561,7 @@ def upload_component(
             finally:
                 shutil.rmtree(tempdir)
         else:
-            archive, manifest = self.pack_component(
+            archive, examples_archive, manifest = self.pack_component(
                 name=name,
                 version=version,
                 dest_dir=dest_dir,
@@ -586,10 +608,15 @@ def callback(monitor: MultipartEncoderMonitor) -> None:
                 memo['progress'] = monitor.bytes_read
 
             if dry_run:
-                job_id = api_client.validate_version(file_path=archive, callback=callback)
+                job_id = api_client.validate_version(
+                    file_path=archive, callback=callback, example_file_path=examples_archive
+                )
             else:
                 job_id = api_client.upload_version(
-                    component_name=component_name, file_path=archive, callback=callback
+                    component_name=component_name,
+                    file_path=archive,
+                    example_file_path=examples_archive,
+                    callback=callback,
                 )
 
             progress_bar.close()

idf_component_manager/core_utils.py

@@ -1,6 +1,5 @@
 # SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-import os
 import re
 import shutil
 import typing as t
@@ -13,8 +12,6 @@
 from idf_component_tools.errors import ComponentModifiedError, FatalError, ModifiedComponent
 from idf_component_tools.file_tools import (
     check_unexpected_component_files,
-    copy_directories,
-    filtered_paths,
 )
 from idf_component_tools.manager import ManifestManager, UploadMode
 from idf_component_tools.manifest import Manifest
@@ -176,45 +173,27 @@ def parse_component_name_spec(
 
 
 def collect_directories(dir_path: Path) -> t.List[str]:
-    directories: t.List[str] = []
     if not dir_path.is_dir():
-        return directories
+        return []
 
-    for directory in os.listdir(str(dir_path)):
-        if directory.startswith('.') or not (dir_path / directory).is_dir():
-            continue
-
-        directories.append(directory)
-
-    return directories
+    return [
+        entry.name
+        for entry in dir_path.iterdir()
+        if entry.is_dir() and not entry.name.startswith('.')
+    ]
 
 
-def detect_duplicate_examples(example_folders, example_path, example_name):
-    for key, value in example_folders.items():
-        if example_name in value:
-            return key, example_path, example_name
-    return
-
-
-def copy_examples_folders(
+def check_examples_folder(
     examples_manifest: t.List[t.Dict[str, str]],
     working_path: Path,
-    dist_dir: Path,
-    use_gitignore: bool = False,
-    include: t.Optional[t.Set[str]] = None,
-    exclude: t.Optional[t.Set[str]] = None,
 ) -> None:
-    examples_path = working_path / 'examples'
-    example_folders = {'examples': collect_directories(examples_path)}
+    example_folders = {'examples': collect_directories(working_path / 'examples')}
     error_paths = []
-    duplicate_paths = []
     for example_info in examples_manifest:
         example_path = example_info['path']
-        example_name = Path(example_path).name
-        full_example_path = working_path / example_path
 
-        if not full_example_path.is_dir():
-            error_paths.append(str(full_example_path))
+        if not (working_path / example_path).is_dir():
+            error_paths.append(str(working_path / example_path))
             continue
 
         if example_path in example_folders.keys():
@@ -223,32 +202,11 @@ def copy_examples_folders(
                 'Please make paths unique and delete duplicate paths'.format(example_path)
             )
 
-        duplicates = detect_duplicate_examples(example_folders, example_path, example_name)
-        if duplicates:
-            duplicate_paths.append(duplicates)
-            continue
-
-        example_folders[example_path] = [example_name]
-
-        paths = filtered_paths(
-            full_example_path, use_gitignore=use_gitignore, include=include, exclude=exclude
-        )
-        copy_directories(str(full_example_path), str(dist_dir / 'examples' / example_name), paths)
+        example_folders[example_path] = [Path(example_path).name]
 
     if error_paths:
         raise FatalError(
             "Example directory doesn't exist: {}.\n"
             'Please check the path of the custom example folder in `examples` field '
             'in `idf_component.yml` file'.format(', '.join(error_paths))
         )
-
-    if duplicate_paths:
-        error_messages = []
-        for first_path, second_path, example_name in duplicate_paths:
-            error_messages.append(
-                f'Examples from "{first_path}" and "{second_path}" '
-                f'have the same name: {example_name}.'
-            )
-        error_messages.append('Please rename one of them, or delete if there are the same')
-
-        raise FatalError('\n'.join(error_messages))

idf_component_tools/build_system_tools.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 """Tools for interaction with IDF build system"""
 
@@ -52,7 +52,7 @@ def get_idf_version():
         idf_version = subprocess.check_output([sys.executable, idf_py_path, '--version'])  # noqa: S603
     except subprocess.CalledProcessError:
         raise RunningEnvironmentError(
-            'Could not get IDF version from calling "idf.py --version".\n' 'idf.py path: {}'.format(
+            'Could not get IDF version from calling "idf.py --version".\nidf.py path: {}'.format(
                 idf_py_path
             )
         )
@@ -70,7 +70,7 @@ def get_idf_version():
         return str(Version.coerce(res[0]))
     else:
         raise RunningEnvironmentError(
-            'Could not parse IDF version from calling "idf.py --version".\n' 'Output: {}'.format(
+            'Could not parse IDF version from calling "idf.py --version".\nOutput: {}'.format(
                 idf_version
             )
         )

idf_component_tools/registry/api_client.py

@@ -1,10 +1,10 @@
-# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 """Classes to work with ESP Component Registry"""
 
-import os
 import typing as t
 from functools import wraps
+from pathlib import Path
 from ssl import SSLEOFError
 
 from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
@@ -112,30 +112,50 @@ def revoke_current_token(self, request: t.Callable) -> None:
         """Revoke current token"""
         request('delete', ['tokens', 'current'])
 
-    def _upload_version_to_endpoint(self, request, file_path, endpoint, callback=None):
-        with open(file_path, 'rb') as file:
-            filename = os.path.basename(file_path)
-
-            encoder = MultipartEncoder({'file': (filename, file, 'application/octet-stream')})
-            headers = {'Content-Type': encoder.content_type}
-            data = MultipartEncoderMonitor(encoder, callback)
+    def _upload_version_to_endpoint(
+        self, request, file_path, example_file_path, endpoint, callback=None
+    ):
+        version_archive_file_handler = open(file_path, 'rb')
+        file_handlers = [version_archive_file_handler]
+        files = {
+            'file': (Path(file_path).name, version_archive_file_handler, 'application/octet-stream')
+        }
+
+        # Handling of example archives defined in the manifest
+        if example_file_path:
+            example_archive_file_handler = open(example_file_path, 'rb')
+            file_handlers.append(example_archive_file_handler)
+            files['example_file'] = (
+                Path(example_file_path).name,
+                example_archive_file_handler,
+                'application/octet-stream',
+            )
 
-            try:
-                return request(
-                    'post',
-                    endpoint,
-                    data=data,
-                    headers=headers,
-                    schema=VersionUpload,
-                    timeout=UPLOAD_COMPONENT_TIMEOUT,
-                )['job_id']
-            # Python 3.10+ can't process 413 error - https://github.com/urllib3/urllib3/issues/2733
-            except (SSLEOFError, ContentTooLargeError):
-                raise APIClientError(
-                    'The component archive exceeds the maximum allowed size. Please consider '
-                    'excluding unnecessary files from your component. If you think your component '
-                    'should be uploaded as it is, please contact [email protected]'
-                )
+        # Encode the archives into a multipart form
+        encoder = MultipartEncoder(files)
+        headers = {'Content-Type': encoder.content_type}
+        data = MultipartEncoderMonitor(encoder, callback)
+
+        try:
+            req = request(
+                'post',
+                endpoint,
+                data=data,
+                headers=headers,
+                schema=VersionUpload,
+                timeout=UPLOAD_COMPONENT_TIMEOUT,
+            )
+            for file_handler in file_handlers:
+                file_handler.close()
+
+            return req['job_id']
+        # Python 3.10+ can't process 413 error - https://github.com/urllib3/urllib3/issues/2733
+        except (SSLEOFError, ContentTooLargeError):
+            raise APIClientError(
+                'The component archive exceeds the maximum allowed size. Please consider '
+                'excluding unnecessary files from your component. If you think your component '
+                'should be uploaded as it is, please contact [email protected]'
+            )
 
     @_request
     def get_component_response(
@@ -147,18 +167,21 @@ def get_component_response(
 
     @auth_required
     @_request
-    def upload_version(self, request, component_name, file_path, callback=None):
+    def upload_version(
+        self, request, component_name, file_path, example_file_path=None, callback=None
+    ):
         return self._upload_version_to_endpoint(
             request,
             file_path,
+            example_file_path,
             ['components', component_name.lower(), 'versions'],
             callback,
         )
 
     @_request
-    def validate_version(self, request, file_path, callback=None):
+    def validate_version(self, request, file_path, example_file_path=None, callback=None):
         return self._upload_version_to_endpoint(
-            request, file_path, ['components', 'validate'], callback
+            request, file_path, example_file_path, ['components', 'validate'], callback
         )
 
     @auth_required

tests/conftest.py

@@ -182,6 +182,15 @@ def release_component_path(fixtures_path):
     )
 
 
+@pytest.fixture(scope='session')
+def cmp_with_example(fixtures_path):
+    return os.path.join(
+        fixtures_path,
+        'components',
+        'cmp_with_example',
+    )
+
+
 @pytest.fixture(scope='session')
 def example_component_path(fixtures_path):
     return os.path.join(