Add support for a list of builders per platform

Author: shanejbrown

buildrunner/config/models.py

@@ -124,7 +124,7 @@ class GlobalConfig(BaseModel, extra="forbid"):
     build_registry: Optional[str] = Field(
         alias="build-registry", default=MP_LOCAL_REGISTRY
     )
-    platform_builders: Optional[Dict[str, str]] = Field(
+    platform_builders: Optional[Union[Dict[str, str], Dict[str, List[str]]]] = Field(
         alias="platform-builders", default=None
     )
     security_scan: GlobalSecurityScanConfig = Field(

buildrunner/docker/multiplatform_image_builder.py

@@ -8,6 +8,7 @@
 import logging
 import os
 import platform as python_platform
+import random
 import re
 import shutil
 import tempfile
@@ -76,7 +77,7 @@ def __init__(
         docker_registry: Optional[str] = None,
         build_registry: Optional[str] = MP_LOCAL_REGISTRY,
         temp_dir: str = os.getcwd(),
-        platform_builders: Optional[Dict[str, str]] = None,
+        platform_builders: Optional[Union[Dict[str, List[str]], Dict[str, str]]] = None,
         cache_builders: Optional[List[str]] = None,
         cache_from: Optional[Union[dict, str]] = None,
         cache_to: Optional[Union[dict, str]] = None,
@@ -311,9 +312,23 @@ def _build_single_image(
             f"'{dockerfile}' ({os.path.exists(dockerfile)}) does not exist!"
         )
 
-        builder = (
+        # Get the builder for the platform
+        builders = (
             self._platform_builders.get(platform) if self._platform_builders else None
         )
+        builder = None
+
+        if builders:
+            if isinstance(builders, str):
+                builder = builders
+            elif isinstance(builders, list):
+                # Randomly select a builder from the list
+                builder = random.choice(builders)
+            else:
+                raise BuildRunnerConfigurationError(
+                    f"Invalid platform builders configuration for platform {platform}"
+                )
+
         LOGGER.debug(f"Building image {image_ref} for platform {platform}")
         LOGGER.info(
             f"Building image for platform {platform} with {builder or 'default'} builder"

docs/global-configuration.rst

@@ -99,8 +99,16 @@ they are used when put into the global configuration file:
   # and therefore this must be configured in buildrunner itself to perform builds
   # across multiple builders for different platforms. Any platform not specified
   # here will use the default configured buildx builder.
+  # 
+  # Each platform can be configured with either a single builder (string) or
+  # a list of builders (array of strings). When using a list, buildrunner will
+  # randomly select one builder for each build.
   platform-builders:
     platform1: builder1
+    platform2:
+      - builder1
+      - builder2
+      - builder3
 
   # Configures caching *for multi-platform builds only*
   docker-build-cache:

examples/build/builders/README.rst

@@ -0,0 +1,73 @@
+==================
+Builders Example
+==================
+
+This example demonstrates how to use Buildrunner with custom builders and random builder selection.
+
+Platform Builders Configuration
+===============================
+
+The ``platform-builders`` configuration allows you to specify which builders to use for different platforms. When multiple builders are available for a platform, Buildrunner will randomly select one to distribute the load.
+
+Configuration Options
+---------------------
+
+1. **Multiple builders per platform** (recommended for load balancing):
+
+   .. code-block:: yaml
+
+       platform-builders:
+         linux/amd64:
+           - builder1
+           - builder2
+           - builder3
+         linux/arm64:
+           - builder4
+           - builder5
+
+2. **Single builder per platform**:
+
+   .. code-block:: yaml
+
+       platform-builders:
+         linux/amd64: builder1
+         linux/arm64: builder2
+
+
+Random Selection
+----------------
+
+When multiple builders are configured for a platform, Buildrunner will randomly select one builder for each build. This helps distribute the build load across available builders and can improve build performance by utilizing multiple build resources.
+
+How to Run
+==========
+
+1. **Create builders**
+
+   From the base directory, run:
+
+   .. code-block:: sh
+
+       python examples/build/builders/create_builders.sh
+
+2. **Run build with example configuration file**
+
+   From the base directory, run the following command:
+
+   .. code-block:: sh
+
+       ./run-buildrunner.sh -f examples/build/builders/buildrunner.yaml -c examples/build/builders/global-config.yaml
+
+   or
+
+   .. code-block:: sh
+
+       ./run-buildrunner.sh -f examples/build/builders/buildrunner.yaml -c examples/build/builders/global-config-list.yaml
+
+3. **Remove builders**
+
+   From the base directory, run:
+
+   .. code-block:: sh
+
+       python examples/build/builders/remove_builders.sh

examples/build/builders/buildrunner.yaml

@@ -0,0 +1,17 @@
+# Example of a buildrunner.yaml file that specifies a multi-platform build step in order to test the builders.
+
+# Execute using:
+# ./run-buildrunner.sh -f examples/build/builders/buildrunner.yaml -c examples/build/builders/global-config-list.yaml
+
+steps:
+  multi-platform-build-step:
+    build:
+      dockerfile: |
+        FROM alpine:latest
+        LABEL custom_label="Buildrunner example label"
+      platforms:
+      - linux/arm/v6
+      - linux/arm/v7
+      - linux/arm/v8
+      - linux/arm64
+      - linux/arm64/v8

examples/build/builders/create_builders.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+docker buildx create --name builder1 --driver docker-container --bootstrap
+docker buildx create --name builder2 --driver docker-container --bootstrap
+docker buildx create --name builder3 --driver docker-container --bootstrap
+
+docker buildx ls --debug

examples/build/builders/global-config-list.yaml

@@ -0,0 +1,20 @@
+# This is a global configuration file that lists the builders for each platform.
+platform-builders:
+  linux/arm/v6:
+    - builder1
+    - builder2
+    - builder3
+  linux/arm/v7:
+    - builder1
+    - builder2
+    - builder3
+  linux/arm/v8:
+    - builder1
+    - builder2
+    - builder3
+  linux/arm64:
+    - builder1
+    - builder2
+    - builder3
+  linux/arm64/v8:
+    - builder1

examples/build/builders/global-config.yaml

@@ -0,0 +1,7 @@
+
+platform-builders:
+  linux/arm/v6: builder1
+  linux/arm/v7: builder2
+  linux/arm/v8: builder3
+  linux/arm64: builder1
+  linux/arm64/v8: builder2

examples/build/builders/remove_builders.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+docker buildx rm builder1
+docker buildx rm builder2
+docker buildx rm builder3

tests/test_config_validation/test_global_config.py

@@ -120,6 +120,36 @@ def fixture_override_master_config_file(tmp_path):
                 "disable-multi-platform:  Input should be a valid boolean, unable to interpret input (bool_parsing)"
             ],
         ),
+        (
+            """
+          platform-builders:
+            linux/amd64:
+              - builder1
+              - builder2
+            linux/arm64:
+              - builder3
+          """,
+            [],
+        ),
+        (
+            """
+          platform-builders:
+            linux/amd64: builder1
+            linux/arm64: builder2
+          """,
+            [],
+        ),
+        (
+            """
+          platform-builders:
+            - builder1
+            - builder2
+          """,
+            [
+                "  platform-builders.dict[str,str]:  Input should be a valid dictionary (dict_type)",
+                "  platform-builders.dict[str,list[str]]:  Input should be a valid dictionary (dict_type)",
+            ],
+        ),
     ],
 )
 def test_config_data(