Pass secrets to buildx builder

Author: shanejbrown

README.rst

@@ -350,6 +350,62 @@ shows the different configuration options available:
         # image is passed to subsequent steps.
         import: path/to/image/archive.tar
 
+        # Specify the secrets that should be used when building your image,
+        # similar to the --secret option used by Docker
+        # More info about secrets: https://docs.docker.com/build/building/secrets/
+        secrets:
+          # Example of a secret that is a file
+          - id=secret1,src=examples/build/secrets/secret1.txt
+          # Example of a secret that is an environment variable
+          - id=secret2,env=SECRET2
+
+
+.. _Build Secrets:
+
+Build Secrets
+=============
+
+Buildrunner supports specifying secrets that should be used when building your image,
+similar to the --secret option used by Docker. This is done by adding the ``secrets``
+section to the ``build`` section. This is a list of secrets that should be used when
+building the image. The string should be in the format of ``id=secret1,src=<location of the file>``
+when the secret is a file or ``id=secret2,env=<environment variable name>`` when the secret is an environment variable.
+This syntax is the same as the syntax used by Docker to build with secrets.
+More info about building with secrets in docker and the syntax of the secret string
+see https://docs.docker.com/build/building/secrets/.
+
+In order to use secrets in buildrunner, you need to do the following:
+
+#. Update the buildrunner configuration file
+    * Set ``use-legacy-builder`` to ``false``
+    * Add the secrets to the ``secrets`` section in the ``build`` section
+#. Update the Dockerfile to use the secrets
+    * Add the ``--mount`` at the beginning of each RUN command that needs the secret
+
+.. code:: yaml
+
+  use-legacy-builder: false
+  steps:
+    build-my-container:
+      build:
+        dockerfile: |
+          FROM alpine:latest
+          RUN --mount=type=secret,id=secret1 \
+              --mount=type=secret,id=secret2 \
+              SECRET1_FILE=/run/secrets/secret1 \
+              SECRET2_VARIABLE=$(cat /run/secrets/secret2) \
+              echo "Using secrets in my build - secret1: $(cat $SECRET1_FILE) secret2: $SECRET2_VARIABLE"
+        secrets:
+          # Example of a secret that is a file
+          - id=secret1,src=examples/build/secrets/secret1.txt
+          # Example of a secret that is an environment variable
+          - id=secret2,env=SECRET2
+
+
+
+
+
+
 
 .. _Running Containers:
 

buildrunner/config/models_step.py

@@ -80,6 +80,7 @@ class StepBuild(StepTask):
     cache_to: Optional[Union[str, Dict[str, str]]] = None
     # import is a python reserved keyword so we need to alias it
     import_param: Optional[str] = Field(alias="import", default=None)
+    secrets: Optional[List[str]] = None
 
 
 class RunAndServicesBase(StepTask):

buildrunner/docker/multiplatform_image_builder.py

@@ -80,6 +80,7 @@ def __init__(
         cache_builders: Optional[List[str]] = None,
         cache_from: Optional[Union[dict, str]] = None,
         cache_to: Optional[Union[dict, str]] = None,
+        secrets: Optional[List[str]] = None,
     ):
         self._docker_registry = docker_registry
         self._build_registry = build_registry
@@ -89,6 +90,7 @@ def __init__(
         self._cache_builders = set(cache_builders if cache_builders else [])
         self._cache_from = cache_from
         self._cache_to = cache_to
+        self._secrets = secrets
         if self._cache_from or self._cache_to:
             LOGGER.info(
                 f'Configuring multiplatform builds to cache from {cache_from} and to {cache_to} '
@@ -197,17 +199,18 @@ def _build_with_inject(
         inject: dict,
         image_ref: str,
         platform: str,
-        path: str,
+        context_path: str,
         dockerfile: str,
         target: str,
         build_args: dict,
         builder: Optional[str],
         cache: bool = False,
         pull: bool = False,
+        secrets: Optional[List[str]] = None,
     ) -> None:
-        if not path or not os.path.isdir(path):
+        if not context_path or not os.path.isdir(context_path):
             LOGGER.warning(
-                f"Failed to inject {inject} for {image_ref} since path {path} isn't a directory."
+                f"Failed to inject {inject} for {image_ref} since path {context_path} isn't a directory."
             )
             return
 
@@ -217,14 +220,14 @@ def _build_with_inject(
         ) as tmp_dir:
             context_dir = os.path.join(tmp_dir, f"{dir_prefix}/")
             shutil.copytree(
-                path,
+                context_path,
                 context_dir,
                 ignore=shutil.ignore_patterns(dir_prefix, ".git"),
                 symlinks=True,
             )
 
             for src, dest in inject.items():
-                src_path = os.path.join(path, src)
+                src_path = os.path.join(context_path, src)
 
                 # Remove '/' prefix
                 if dest.startswith("/"):
@@ -265,6 +268,7 @@ def _build_with_inject(
                 cache=cache,
                 pull=pull,
                 stream_logs=True,
+                secrets=secrets,
                 **self._get_build_cache_options(builder),
             )
             self._log_buildx(logs_itr, platform)
@@ -294,6 +298,7 @@ def _build_single_image(
         inject: dict,
         cache: bool = False,
         pull: bool = False,
+        secrets: Optional[List[str]] = None,
     ) -> None:
         """
         Builds a single image for the given platform.
@@ -307,6 +312,7 @@ def _build_single_image(
             target (str): The name of the stage to build in a multi-stage Dockerfile
             build_args (dict): The build args to pass to docker.
             inject (dict): The files to inject into the build context.
+            secrets (List[str]): The secrets to pass to docker.
         """
         assert os.path.isdir(path) and os.path.exists(dockerfile), (
             f"Either path {path} ({os.path.isdir(path)}) or file "
@@ -321,32 +327,37 @@ def _build_single_image(
             f"Building image for platform {platform} with {builder or 'default'} builder"
         )
 
+        # Build kwargs for the buildx build command
+        build_kwargs = {
+            "builder": builder,
+            "cache": cache,
+            "context_path": path,
+            "pull": pull,
+            "target": target,
+        }
+
+        # Only pass secrets if they are provided
+        if secrets:
+            build_kwargs["secrets"] = secrets
+
         if inject and isinstance(inject, dict):
             self._build_with_inject(
                 inject=inject,
                 image_ref=image_ref,
                 platform=platform,
-                path=path,
                 dockerfile=dockerfile,
-                target=target,
                 build_args=build_args,
-                builder=builder,
-                cache=cache,
-                pull=pull,
+                **build_kwargs,
             )
         else:
             logs_itr = docker.buildx.build(
-                path,
                 tags=[image_ref],
                 platforms=[platform],
                 load=True,
                 file=dockerfile,
-                target=target,
                 build_args=build_args,
-                builder=builder,
-                cache=cache,
-                pull=pull,
                 stream_logs=True,
+                **build_kwargs,
                 **self._get_build_cache_options(builder),
             )
             self._log_buildx(logs_itr, platform)
@@ -398,11 +409,12 @@ def build_multiple_images(
         path: str = ".",
         file: str = "Dockerfile",
         target: Optional[str] = None,
-        use_threading: bool = True,
+        use_threading: bool = False,
         build_args: dict = None,
         inject: dict = None,
         cache: bool = False,
         pull: bool = False,
+        secrets: Optional[List[str]] = None,
     ) -> BuiltImageInfo:
         """
         Builds multiple images for the given platforms. One image will be built for each platform.
@@ -498,6 +510,7 @@ def build_multiple_images(
                 inject,
                 cache,
                 pull,
+                secrets,
             )
             LOGGER.debug(f"Building {repo} for {platform}")
             if use_threading:

buildrunner/steprunner/tasks/build.py

@@ -238,6 +238,7 @@ def run(self, context):
                     inject=self.to_inject,
                     cache=not self.nocache,
                     pull=self.pull,
+                    secrets=self.step.secrets,
                 )
 
                 # Set expected number of platforms

examples/build/secrets/buildrunner.yaml

@@ -0,0 +1,21 @@
+# In order to use secrets, you need to set use-legacy-builder to false in the config file
+# To run this example, you need to set the SECRET_PASSWORD environment variable
+# and run the example with the following command:
+# SECRET2=my_secret ./run-buildrunner.sh -f examples/build/secrets/buildrunner.yaml
+# More info about secrets: https://docs.docker.com/build/building/secrets/
+use-legacy-builder: false
+steps:
+  simple-build-step:
+    build:
+      dockerfile: |
+        FROM alpine:latest
+        RUN --mount=type=secret,id=secret1 \
+            --mount=type=secret,id=secret2 \
+            SECRET1_FILE=/run/secrets/secret1 \
+            SECRET2_VARIABLE=$(cat /run/secrets/secret2) \
+            echo "Using secrets in my build - secret1: $(cat $SECRET1_FILE) secret2: $SECRET2_VARIABLE"
+      secrets:
+        # Example of a secret that is a file
+        - id=secret1,src=examples/build/secrets/secret1.txt
+        # Example of a secret that is an environment variable
+        - id=secret2,env=SECRET2

examples/build/secrets/secret1.txt

@@ -0,0 +1 @@
+testuser123