Skip to content

Commit befd6d9

Browse files
shanejbrownshanejbrown
authored
Merge pull request #214 from shanejbrown/buildx-builders
Buildx builders
2 parents 77254c0 + e99fd06 commit befd6d9

File tree

10 files changed

+185
-3
lines changed

10 files changed

+185
-3
lines changed

‎buildrunner/config/models.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ class GlobalConfig(BaseModel, extra="forbid"):
124124
build_registry: Optional[str] = Field(
125125
alias="build-registry", default=MP_LOCAL_REGISTRY
126126
)
127-
platform_builders: Optional[Dict[str, str]] = Field(
127+
platform_builders: Optional[Union[Dict[str, str], Dict[str, List[str]]]] = Field(
128128
alias="platform-builders", default=None
129129
)
130130
security_scan: GlobalSecurityScanConfig = Field(

‎buildrunner/docker/multiplatform_image_builder.py‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import logging
99
import os
1010
import platform as python_platform
11+
import random
1112
import re
1213
import shutil
1314
import tempfile
@@ -76,7 +77,7 @@ def __init__(
7677
docker_registry: Optional[str] = None,
7778
build_registry: Optional[str] = MP_LOCAL_REGISTRY,
7879
temp_dir: str = os.getcwd(),
79-
platform_builders: Optional[Dict[str, str]] = None,
80+
platform_builders: Optional[Union[Dict[str, List[str]], Dict[str, str]]] = None,
8081
cache_builders: Optional[List[str]] = None,
8182
cache_from: Optional[Union[dict, str]] = None,
8283
cache_to: Optional[Union[dict, str]] = None,
@@ -311,9 +312,23 @@ def _build_single_image(
311312
f"'{dockerfile}' ({os.path.exists(dockerfile)}) does not exist!"
312313
)
313314

314-
builder = (
315+
# Get the builder for the platform
316+
builders = (
315317
self._platform_builders.get(platform) if self._platform_builders else None
316318
)
319+
builder = None
320+
321+
if builders:
322+
if isinstance(builders, str):
323+
builder = builders
324+
elif isinstance(builders, list):
325+
# Randomly select a builder from the list
326+
builder = random.choice(builders)
327+
else:
328+
raise BuildRunnerConfigurationError(
329+
f"Invalid platform builders configuration for platform {platform}"
330+
)
331+
317332
LOGGER.debug(f"Building image {image_ref} for platform {platform}")
318333
LOGGER.info(
319334
f"Building image for platform {platform} with {builder or 'default'} builder"

‎docs/global-configuration.rst‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,8 +99,16 @@ they are used when put into the global configuration file:
9999
# and therefore this must be configured in buildrunner itself to perform builds
100100
# across multiple builders for different platforms. Any platform not specified
101101
# here will use the default configured buildx builder.
102+
#
103+
# Each platform can be configured with either a single builder (string) or
104+
# a list of builders (array of strings). When using a list, buildrunner will
105+
# randomly select one builder for each build.
102106
platform-builders:
103107
platform1: builder1
108+
platform2:
109+
- builder1
110+
- builder2
111+
- builder3
104112
105113
# Configures caching *for multi-platform builds only*
106114
docker-build-cache:

‎examples/build/builders/README.rst‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
==================
2+
Builders Example
3+
==================
4+
5+
This example demonstrates how to use Buildrunner with custom builders and random builder selection.
6+
7+
Platform Builders Configuration
8+
===============================
9+
10+
The ``platform-builders`` configuration allows you to specify which builders to use for different platforms. When multiple builders are available for a platform, Buildrunner will randomly select one to distribute the load.
11+
12+
Configuration Options
13+
---------------------
14+
15+
1. **Multiple builders per platform** (recommended for load balancing):
16+
17+
.. code-block:: yaml
18+
19+
platform-builders:
20+
linux/amd64:
21+
- builder1
22+
- builder2
23+
- builder3
24+
linux/arm64:
25+
- builder4
26+
- builder5
27+
28+
2. **Single builder per platform**:
29+
30+
.. code-block:: yaml
31+
32+
platform-builders:
33+
linux/amd64: builder1
34+
linux/arm64: builder2
35+
36+
37+
Random Selection
38+
----------------
39+
40+
When multiple builders are configured for a platform, Buildrunner will randomly select one builder for each build. This helps distribute the build load across available builders and can improve build performance by utilizing multiple build resources.
41+
42+
How to Run
43+
==========
44+
45+
1. **Create builders**
46+
47+
From the base directory, run:
48+
49+
.. code-block:: sh
50+
51+
python examples/build/builders/create_builders.sh
52+
53+
2. **Run build with example configuration file**
54+
55+
From the base directory, run the following command:
56+
57+
.. code-block:: sh
58+
59+
./run-buildrunner.sh -f examples/build/builders/buildrunner.yaml -c examples/build/builders/global-config.yaml
60+
61+
or
62+
63+
.. code-block:: sh
64+
65+
./run-buildrunner.sh -f examples/build/builders/buildrunner.yaml -c examples/build/builders/global-config-list.yaml
66+
67+
3. **Remove builders**
68+
69+
From the base directory, run:
70+
71+
.. code-block:: sh
72+
73+
python examples/build/builders/remove_builders.sh

‎examples/build/builders/buildrunner.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Example of a buildrunner.yaml file that specifies a multi-platform build step in order to test the builders.
2+
3+
# Execute using:
4+
# ./run-buildrunner.sh -f examples/build/builders/buildrunner.yaml -c examples/build/builders/global-config-list.yaml
5+
6+
steps:
7+
multi-platform-build-step:
8+
build:
9+
dockerfile: |
10+
FROM alpine:latest
11+
LABEL custom_label="Buildrunner example label"
12+
platforms:
13+
- linux/arm/v6
14+
- linux/arm/v7
15+
- linux/arm/v8
16+
- linux/arm64
17+
- linux/arm64/v8

‎examples/build/builders/create_builders.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
#!/bin/bash
2+
3+
docker buildx create --name builder1 --driver docker-container --bootstrap
4+
docker buildx create --name builder2 --driver docker-container --bootstrap
5+
docker buildx create --name builder3 --driver docker-container --bootstrap
6+
7+
docker buildx ls --debug

‎examples/build/builders/global-config-list.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# This is a global configuration file that lists the builders for each platform.
2+
platform-builders:
3+
linux/arm/v6:
4+
- builder1
5+
- builder2
6+
- builder3
7+
linux/arm/v7:
8+
- builder1
9+
- builder2
10+
- builder3
11+
linux/arm/v8:
12+
- builder1
13+
- builder2
14+
- builder3
15+
linux/arm64:
16+
- builder1
17+
- builder2
18+
- builder3
19+
linux/arm64/v8:
20+
- builder1

‎examples/build/builders/global-config.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
2+
platform-builders:
3+
linux/arm/v6: builder1
4+
linux/arm/v7: builder2
5+
linux/arm/v8: builder3
6+
linux/arm64: builder1
7+
linux/arm64/v8: builder2

‎examples/build/builders/remove_builders.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
3+
docker buildx rm builder1
4+
docker buildx rm builder2
5+
docker buildx rm builder3

‎tests/test_config_validation/test_global_config.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,36 @@ def fixture_override_master_config_file(tmp_path):
120120
"disable-multi-platform: Input should be a valid boolean, unable to interpret input (bool_parsing)"
121121
],
122122
),
123+
(
124+
"""
125+
platform-builders:
126+
linux/amd64:
127+
- builder1
128+
- builder2
129+
linux/arm64:
130+
- builder3
131+
""",
132+
[],
133+
),
134+
(
135+
"""
136+
platform-builders:
137+
linux/amd64: builder1
138+
linux/arm64: builder2
139+
""",
140+
[],
141+
),
142+
(
143+
"""
144+
platform-builders:
145+
- builder1
146+
- builder2
147+
""",
148+
[
149+
" platform-builders.dict[str,str]: Input should be a valid dictionary (dict_type)",
150+
" platform-builders.dict[str,list[str]]: Input should be a valid dictionary (dict_type)",
151+
],
152+
),
123153
],
124154
)
125155
def test_config_data(

0 commit comments

Comments
 (0)