diff --git a/buildrunner/config/models.py b/buildrunner/config/models.py index 9c37f47d..5822515f 100644 --- a/buildrunner/config/models.py +++ b/buildrunner/config/models.py @@ -124,7 +124,7 @@ class GlobalConfig(BaseModel, extra="forbid"): build_registry: Optional[str] = Field( alias="build-registry", default=MP_LOCAL_REGISTRY ) - platform_builders: Optional[Dict[str, str]] = Field( + platform_builders: Optional[Union[Dict[str, str], Dict[str, List[str]]]] = Field( alias="platform-builders", default=None ) security_scan: GlobalSecurityScanConfig = Field( diff --git a/buildrunner/docker/multiplatform_image_builder.py b/buildrunner/docker/multiplatform_image_builder.py index c4c73a75..b8abbbe6 100644 --- a/buildrunner/docker/multiplatform_image_builder.py +++ b/buildrunner/docker/multiplatform_image_builder.py @@ -8,6 +8,7 @@ import logging import os import platform as python_platform +import random import re import shutil import tempfile @@ -76,7 +77,7 @@ def __init__( docker_registry: Optional[str] = None, build_registry: Optional[str] = MP_LOCAL_REGISTRY, temp_dir: str = os.getcwd(), - platform_builders: Optional[Dict[str, str]] = None, + platform_builders: Optional[Union[Dict[str, List[str]], Dict[str, str]]] = None, cache_builders: Optional[List[str]] = None, cache_from: Optional[Union[dict, str]] = None, cache_to: Optional[Union[dict, str]] = None, @@ -311,9 +312,23 @@ def _build_single_image( f"'{dockerfile}' ({os.path.exists(dockerfile)}) does not exist!" ) - builder = ( + # Get the builder for the platform + builders = ( self._platform_builders.get(platform) if self._platform_builders else None ) + builder = None + + if builders: + if isinstance(builders, str): + builder = builders + elif isinstance(builders, list): + # Randomly select a builder from the list + builder = random.choice(builders) + else: + raise BuildRunnerConfigurationError( + f"Invalid platform builders configuration for platform {platform}" + ) + LOGGER.debug(f"Building image {image_ref} for platform {platform}") LOGGER.info( f"Building image for platform {platform} with {builder or 'default'} builder" diff --git a/docs/global-configuration.rst b/docs/global-configuration.rst index 437441bc..c67706b5 100644 --- a/docs/global-configuration.rst +++ b/docs/global-configuration.rst @@ -99,8 +99,16 @@ they are used when put into the global configuration file: # and therefore this must be configured in buildrunner itself to perform builds # across multiple builders for different platforms. Any platform not specified # here will use the default configured buildx builder. + # + # Each platform can be configured with either a single builder (string) or + # a list of builders (array of strings). When using a list, buildrunner will + # randomly select one builder for each build. platform-builders: platform1: builder1 + platform2: + - builder1 + - builder2 + - builder3 # Configures caching *for multi-platform builds only* docker-build-cache: diff --git a/examples/build/builders/README.rst b/examples/build/builders/README.rst new file mode 100644 index 00000000..d37a6adb --- /dev/null +++ b/examples/build/builders/README.rst @@ -0,0 +1,73 @@ +================== +Builders Example +================== + +This example demonstrates how to use Buildrunner with custom builders and random builder selection. + +Platform Builders Configuration +=============================== + +The ``platform-builders`` configuration allows you to specify which builders to use for different platforms. When multiple builders are available for a platform, Buildrunner will randomly select one to distribute the load. + +Configuration Options +--------------------- + +1. **Multiple builders per platform** (recommended for load balancing): + + .. code-block:: yaml + + platform-builders: + linux/amd64: + - builder1 + - builder2 + - builder3 + linux/arm64: + - builder4 + - builder5 + +2. **Single builder per platform**: + + .. code-block:: yaml + + platform-builders: + linux/amd64: builder1 + linux/arm64: builder2 + + +Random Selection +---------------- + +When multiple builders are configured for a platform, Buildrunner will randomly select one builder for each build. This helps distribute the build load across available builders and can improve build performance by utilizing multiple build resources. + +How to Run +========== + +1. **Create builders** + + From the base directory, run: + + .. code-block:: sh + + python examples/build/builders/create_builders.sh + +2. **Run build with example configuration file** + + From the base directory, run the following command: + + .. code-block:: sh + + ./run-buildrunner.sh -f examples/build/builders/buildrunner.yaml -c examples/build/builders/global-config.yaml + + or + + .. code-block:: sh + + ./run-buildrunner.sh -f examples/build/builders/buildrunner.yaml -c examples/build/builders/global-config-list.yaml + +3. **Remove builders** + + From the base directory, run: + + .. code-block:: sh + + python examples/build/builders/remove_builders.sh diff --git a/examples/build/builders/buildrunner.yaml b/examples/build/builders/buildrunner.yaml new file mode 100644 index 00000000..ed994807 --- /dev/null +++ b/examples/build/builders/buildrunner.yaml @@ -0,0 +1,17 @@ +# Example of a buildrunner.yaml file that specifies a multi-platform build step in order to test the builders. + +# Execute using: +# ./run-buildrunner.sh -f examples/build/builders/buildrunner.yaml -c examples/build/builders/global-config-list.yaml + +steps: + multi-platform-build-step: + build: + dockerfile: | + FROM alpine:latest + LABEL custom_label="Buildrunner example label" + platforms: + - linux/arm/v6 + - linux/arm/v7 + - linux/arm/v8 + - linux/arm64 + - linux/arm64/v8 diff --git a/examples/build/builders/create_builders.sh b/examples/build/builders/create_builders.sh new file mode 100755 index 00000000..c85d1bc3 --- /dev/null +++ b/examples/build/builders/create_builders.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +docker buildx create --name builder1 --driver docker-container --bootstrap +docker buildx create --name builder2 --driver docker-container --bootstrap +docker buildx create --name builder3 --driver docker-container --bootstrap + +docker buildx ls --debug \ No newline at end of file diff --git a/examples/build/builders/global-config-list.yaml b/examples/build/builders/global-config-list.yaml new file mode 100644 index 00000000..0e89878a --- /dev/null +++ b/examples/build/builders/global-config-list.yaml @@ -0,0 +1,20 @@ +# This is a global configuration file that lists the builders for each platform. +platform-builders: + linux/arm/v6: + - builder1 + - builder2 + - builder3 + linux/arm/v7: + - builder1 + - builder2 + - builder3 + linux/arm/v8: + - builder1 + - builder2 + - builder3 + linux/arm64: + - builder1 + - builder2 + - builder3 + linux/arm64/v8: + - builder1 \ No newline at end of file diff --git a/examples/build/builders/global-config.yaml b/examples/build/builders/global-config.yaml new file mode 100644 index 00000000..432cd510 --- /dev/null +++ b/examples/build/builders/global-config.yaml @@ -0,0 +1,7 @@ + +platform-builders: + linux/arm/v6: builder1 + linux/arm/v7: builder2 + linux/arm/v8: builder3 + linux/arm64: builder1 + linux/arm64/v8: builder2 \ No newline at end of file diff --git a/examples/build/builders/remove_builders.sh b/examples/build/builders/remove_builders.sh new file mode 100755 index 00000000..3dc13e0c --- /dev/null +++ b/examples/build/builders/remove_builders.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +docker buildx rm builder1 +docker buildx rm builder2 +docker buildx rm builder3 \ No newline at end of file diff --git a/tests/test_config_validation/test_global_config.py b/tests/test_config_validation/test_global_config.py index 1b9b8d0a..cb52dde6 100644 --- a/tests/test_config_validation/test_global_config.py +++ b/tests/test_config_validation/test_global_config.py @@ -120,6 +120,36 @@ def fixture_override_master_config_file(tmp_path): "disable-multi-platform: Input should be a valid boolean, unable to interpret input (bool_parsing)" ], ), + ( + """ + platform-builders: + linux/amd64: + - builder1 + - builder2 + linux/arm64: + - builder3 + """, + [], + ), + ( + """ + platform-builders: + linux/amd64: builder1 + linux/arm64: builder2 + """, + [], + ), + ( + """ + platform-builders: + - builder1 + - builder2 + """, + [ + " platform-builders.dict[str,str]: Input should be a valid dictionary (dict_type)", + " platform-builders.dict[str,list[str]]: Input should be a valid dictionary (dict_type)", + ], + ), ], ) def test_config_data(