Merge pull request #202 from shanejbrown/secrets

shanejbrown · web-flow · commit 3a7ce7c587db · 2025-04-14T16:09:45.000-06:00
Add support for buildx build secrets
diff --git a/README.rst b/README.rst
@@ -350,6 +350,60 @@ shows the different configuration options available:
         # image is passed to subsequent steps.
         import: path/to/image/archive.tar
 
+        # Specify the secrets that should be used when building your image,
+        # similar to the --secret option used by Docker
+        # More info about secrets: https://docs.docker.com/build/building/secrets/
+        secrets:
+          # Example of a secret that is a file
+          - id=secret1,src=<path to the secret file>
+          # Example of a secret that is an environment variable
+          - id=secret2,env=<environment variable name>
+
+.. _Build Secrets:
+
+Build Secrets
+=============
+
+Buildrunner supports specifying secrets that should be used when building your image,
+similar to the --secret option used by Docker. This is done by adding the ``secrets``
+section to the ``build`` section. This is a list of secrets that should be used when
+building the image. The string should be in the format of ``id=secret1,src=<location of the file>``
+when the secret is a file or ``id=secret2,env=<environment variable name>`` when the secret is an environment variable.
+This syntax is the same as the syntax used by Docker to build with secrets.
+More info about building with secrets in docker and the syntax of the secret string
+see https://docs.docker.com/build/building/secrets/.
+
+In order to use secrets in buildrunner, you need to do the following:
+
+#. Update the buildrunner configuration file
+    * Set ``use-legacy-builder`` to ``false`` or add ``platforms`` to the ``build`` section
+    * Add the secrets to the ``secrets`` section in the ``build`` section
+#. Update the Dockerfile to use the secrets
+    * Add the ``--mount`` at the beginning of each RUN command that needs the secret
+
+.. code:: yaml
+
+  use-legacy-builder: false
+  steps:
+    build-my-container:
+      build:
+        dockerfile: |
+          FROM alpine:latest
+          # Using secrets inline
+          RUN --mount=type=secret,id=secret1 \
+              --mount=type=secret,id=secret2 \
+              echo Using secrets in my build - secret1 file located at /run/secrets/secret1 with contents $(cat /run/secrets/secret1) and secret2=$(cat /run/secrets/secret2)
+          # Using secrets in environment variables
+          RUN --mount=type=secret,id=secret1 \
+              --mount=type=secret,id=secret2 \
+              SECRET1_FILE=/run/secrets/secret1 \
+              SECRET2_VARIABLE=$(cat /run/secrets/secret2) \
+              && echo Using secrets in my build - secret1 file located at $SECRET1_FILE with contents $(cat $SECRET1_FILE) and secret2=$SECRET2_VARIABLE
+        secrets:
+          # Example of a secret that is a file
+          - id=secret1,src=examples/build/secrets/secret1.txt
+          # Example of a secret that is an environment variable
+          - id=secret2,env=SECRET2
 
 .. _Running Containers:
 
diff --git a/buildrunner/config/models.py b/buildrunner/config/models.py
@@ -150,7 +150,7 @@ class Config(BaseModel, extra="forbid"):
 
     @field_validator("steps")
     @classmethod
-    def validate_steps(cls, vals) -> None:
+    def validate_steps(cls, vals, info) -> None:
         """
         Validate the config file
 
@@ -161,13 +161,26 @@ def validate_steps(cls, vals) -> None:
         if not vals:
             raise ValueError('The "steps" configuration was not provided')
 
-        # Checks to see if there is a mutli-platform build step in the config
+        # Checks steps for mutli-platform or secrets
         has_multi_platform_build = False
+
+        #  Check for multi-platform builds and secrets validation
         for step in vals.values():
             has_multi_platform_build = (
                 has_multi_platform_build or step.is_multi_platform()
             )
 
+            # If the step has secrets and the builder is legacy or no platforms are set for the step, raise an error
+            if (
+                step.has_secrets()
+                and not step.is_multi_platform()
+                and info.data.get("use_legacy_builder")
+            ):
+                raise ValueError(
+                    "Build secrets are not supported with the legacy builder. Please set use-legacy-builder to false"
+                    " or add platforms to the build section in order to use secrets in your build."
+                )
+
         if has_multi_platform_build:
             mp_push_tags = set()
             validate_multiplatform_build(vals, mp_push_tags)
diff --git a/buildrunner/config/models_step.py b/buildrunner/config/models_step.py
@@ -80,6 +80,7 @@ class StepBuild(StepTask):
     cache_to: Optional[Union[str, Dict[str, str]]] = None
     # import is a python reserved keyword so we need to alias it
     import_param: Optional[str] = Field(alias="import", default=None)
+    secrets: Optional[List[str]] = None
 
 
 class RunAndServicesBase(StepTask):
@@ -242,3 +243,9 @@ def is_multi_platform(self):
         Check if the step is a multi-platform build step
         """
         return self.build and self.build.platforms is not None
+
+    def has_secrets(self):
+        """
+        Check if the step has secrets
+        """
+        return self.build and self.build.secrets is not None
diff --git a/buildrunner/docker/multiplatform_image_builder.py b/buildrunner/docker/multiplatform_image_builder.py
@@ -80,6 +80,7 @@ def __init__(
         cache_builders: Optional[List[str]] = None,
         cache_from: Optional[Union[dict, str]] = None,
         cache_to: Optional[Union[dict, str]] = None,
+        secrets: Optional[List[str]] = None,
     ):
         self._docker_registry = docker_registry
         self._build_registry = build_registry
@@ -89,6 +90,7 @@ def __init__(
         self._cache_builders = set(cache_builders if cache_builders else [])
         self._cache_from = cache_from
         self._cache_to = cache_to
+        self._secrets = secrets
         if self._cache_from or self._cache_to:
             LOGGER.info(
                 f'Configuring multiplatform builds to cache from {cache_from} and to {cache_to} '
@@ -196,14 +198,11 @@ def _build_with_inject(
         self,
         inject: dict,
         image_ref: str,
-        platform: str,
         path: str,
+        platform: str,
         dockerfile: str,
-        target: str,
         build_args: dict,
-        builder: Optional[str],
-        cache: bool = False,
-        pull: bool = False,
+        build_kwargs: dict,
     ) -> None:
         if not path or not os.path.isdir(path):
             LOGGER.warning(
@@ -256,16 +255,13 @@ def _build_with_inject(
 
             logs_itr = docker.buildx.build(
                 context_dir,
-                tags=[image_ref],
-                platforms=[platform],
-                load=True,
-                target=target,
-                builder=builder,
                 build_args=build_args,
-                cache=cache,
-                pull=pull,
+                load=True,
+                platforms=[platform],
                 stream_logs=True,
-                **self._get_build_cache_options(builder),
+                tags=[image_ref],
+                **build_kwargs,
+                **self._get_build_cache_options(build_kwargs.get("builder")),
             )
             self._log_buildx(logs_itr, platform)
 
@@ -294,6 +290,7 @@ def _build_single_image(
         inject: dict,
         cache: bool = False,
         pull: bool = False,
+        secrets: Optional[List[str]] = None,
     ) -> None:
         """
         Builds a single image for the given platform.
@@ -307,6 +304,7 @@ def _build_single_image(
             target (str): The name of the stage to build in a multi-stage Dockerfile
             build_args (dict): The build args to pass to docker.
             inject (dict): The files to inject into the build context.
+            secrets (List[str]): The secrets to pass to docker.
         """
         assert os.path.isdir(path) and os.path.exists(dockerfile), (
             f"Either path {path} ({os.path.isdir(path)}) or file "
@@ -321,18 +319,28 @@ def _build_single_image(
             f"Building image for platform {platform} with {builder or 'default'} builder"
         )
 
+        # Build kwargs for the buildx build command
+        build_kwargs = {}
+        if builder:
+            build_kwargs["builder"] = builder
+        if cache:
+            build_kwargs["cache"] = cache
+        if pull:
+            build_kwargs["pull"] = pull
+        if secrets:
+            build_kwargs["secrets"] = secrets
+        if target:
+            build_kwargs["target"] = target
+
         if inject and isinstance(inject, dict):
             self._build_with_inject(
+                path=path,
                 inject=inject,
                 image_ref=image_ref,
                 platform=platform,
-                path=path,
                 dockerfile=dockerfile,
-                target=target,
                 build_args=build_args,
-                builder=builder,
-                cache=cache,
-                pull=pull,
+                build_kwargs=build_kwargs,
             )
         else:
             logs_itr = docker.buildx.build(
@@ -341,12 +349,9 @@ def _build_single_image(
                 platforms=[platform],
                 load=True,
                 file=dockerfile,
-                target=target,
                 build_args=build_args,
-                builder=builder,
-                cache=cache,
-                pull=pull,
                 stream_logs=True,
+                **build_kwargs,
                 **self._get_build_cache_options(builder),
             )
             self._log_buildx(logs_itr, platform)
@@ -403,6 +408,7 @@ def build_multiple_images(
         inject: dict = None,
         cache: bool = False,
         pull: bool = False,
+        secrets: Optional[List[str]] = None,
     ) -> BuiltImageInfo:
         """
         Builds multiple images for the given platforms. One image will be built for each platform.
@@ -498,6 +504,7 @@ def build_multiple_images(
                 inject,
                 cache,
                 pull,
+                secrets,
             )
             LOGGER.debug(f"Building {repo} for {platform}")
             if use_threading:
diff --git a/buildrunner/steprunner/tasks/build.py b/buildrunner/steprunner/tasks/build.py
@@ -238,6 +238,7 @@ def run(self, context):
                     inject=self.to_inject,
                     cache=not self.nocache,
                     pull=self.pull,
+                    secrets=self.step.secrets,
                 )
 
                 # Set expected number of platforms
diff --git a/examples/build/secrets/buildrunner.yaml b/examples/build/secrets/buildrunner.yaml
@@ -0,0 +1,27 @@
+# In order to use secrets, you need to set use-legacy-builder to false in the config file
+# To run this example, you need to set the SECRET_PASSWORD environment variable
+# and run the example with the following command:
+# SECRET2=my_secret ./run-buildrunner.sh -f examples/build/secrets/buildrunner.yaml
+# More info about secrets: https://docs.docker.com/build/building/secrets/
+use-legacy-builder: false
+steps:
+  simple-build-step:
+    build:
+      no-cache: true
+      dockerfile: |
+        FROM alpine:latest
+        # Using secrets inline
+        RUN --mount=type=secret,id=secret1 \
+            --mount=type=secret,id=secret2 \
+            echo Using secrets in my build - secret1 file located at /run/secrets/secret1 with contents $(cat /run/secrets/secret1) and secret2=$(cat /run/secrets/secret2)
+        # Using secrets in environment variables
+        RUN --mount=type=secret,id=secret1 \
+            --mount=type=secret,id=secret2 \
+            SECRET1_FILE=/run/secrets/secret1 \
+            SECRET2_VARIABLE=$(cat /run/secrets/secret2) \
+            && echo Using secrets in my build - secret1 file located at $SECRET1_FILE with contents $(cat $SECRET1_FILE) and secret2=$SECRET2_VARIABLE
+      secrets:
+        # Example of a secret that is a file
+        - id=secret1,src=examples/build/secrets/secret1.txt
+        # Example of a secret that is an environment variable
+        - id=secret2,env=SECRET2
diff --git a/examples/build/secrets/platforms-buildrunner.yaml b/examples/build/secrets/platforms-buildrunner.yaml
@@ -0,0 +1,29 @@
+# In order to use secrets, you need to set use-legacy-builder to false in the config file OR
+# add platforms to the build section
+# To run this example, you need to set the SECRET_PASSWORD environment variable
+# and run the example with the following command:
+# SECRET2=my_secret ./run-buildrunner.sh -f examples/build/secrets/platforms-buildrunner.yaml
+# More info about secrets: https://docs.docker.com/build/building/secrets/
+steps:
+  simple-build-step:
+    build:
+      dockerfile: |
+        FROM alpine:latest
+        # Using secrets inline
+        RUN --mount=type=secret,id=secret1 \
+            --mount=type=secret,id=secret2 \
+            echo Using secrets in my build - secret1 file located at /run/secrets/secret1 with contents $(cat /run/secrets/secret1) and secret2=$(cat /run/secrets/secret2)
+        # Using secrets in environment variables
+        RUN --mount=type=secret,id=secret1 \
+            --mount=type=secret,id=secret2 \
+            SECRET1_FILE=/run/secrets/secret1 \
+            SECRET2_VARIABLE=$(cat /run/secrets/secret2) \
+            && echo Using secrets in my build - secret1 file located at $SECRET1_FILE with contents $(cat $SECRET1_FILE) and secret2=$SECRET2_VARIABLE
+      secrets:
+        # Example of a secret that is a file
+        - id=secret1,src=examples/build/secrets/secret1.txt
+        # Example of a secret that is an environment variable
+        - id=secret2,env=SECRET2
+      platforms:
+        - linux/amd64
+        - linux/arm64
diff --git a/examples/build/secrets/secret1.txt b/examples/build/secrets/secret1.txt
@@ -0,0 +1 @@
+testuser123 
diff --git a/tests/test_buildrunner_files.py b/tests/test_buildrunner_files.py
@@ -124,6 +124,8 @@ def _get_example_runs(test_dir: str) -> List[Tuple[str, str, Optional[List[str]]
     excluded_example_files = [
         "examples/build/import/buildrunner.yaml",
         "examples/run/caches/buildrunner.yaml",
+        # This file is not supported in the github actions runner
+        "examples/build/secrets/platforms-buildrunner.yaml",
     ]
 
     # Walk through the examples directory and find all files ending with buildrunner.yaml
diff --git a/tests/test_multiplatform.py b/tests/test_multiplatform.py
@@ -478,12 +478,8 @@ def test_build_multiple_builds(
             load=True,
             file=f"{test_path}/Dockerfile",
             build_args={"DOCKER_REGISTRY": None},
-            builder=None,
-            cache=False,
             cache_from=None,
             cache_to=None,
-            pull=False,
-            target=None,
             stream_logs=True,
         ),
         call(
@@ -493,12 +489,8 @@ def test_build_multiple_builds(
             load=True,
             file=f"{test_path}/Dockerfile",
             build_args={"DOCKER_REGISTRY": None},
-            builder=None,
-            cache=False,
             cache_from=None,
             cache_to=None,
-            pull=False,
-            target=None,
             stream_logs=True,
         ),
         call(
@@ -508,12 +500,8 @@ def test_build_multiple_builds(
             load=True,
             file=f"{test_path}/Dockerfile",
             build_args={"DOCKER_REGISTRY": None},
-            builder=None,
-            cache=False,
             cache_from=None,
             cache_to=None,
-            pull=False,
-            target=None,
             stream_logs=True,
         ),
         call(
@@ -523,12 +511,8 @@ def test_build_multiple_builds(
             load=True,
             file=f"{test_path}/Dockerfile",
             build_args={"DOCKER_REGISTRY": None},
-            builder=None,
-            cache=False,
             cache_from=None,
             cache_to=None,
-            pull=False,
-            target=None,
             stream_logs=True,
         ),
     ]

Original file line number	Diff line number	Diff line change
`@@ -238,6 +238,7 @@ def run(self, context):`
`238`	`238`	`inject=self.to_inject,`
`239`	`239`	`cache=not self.nocache,`
`240`	`240`	`pull=self.pull,`
	`241`	`+ secrets=self.step.secrets,`
`241`	`242`	`)`
`242`	`243`
`243`	`244`	`# Set expected number of platforms`
Original file line number	Diff line number	Diff line change
`@@ -124,6 +124,8 @@ def _get_example_runs(test_dir: str) -> List[Tuple[str, str, Optional[List[str]]`
`124`	`124`	`excluded_example_files = [`
`125`	`125`	`"examples/build/import/buildrunner.yaml",`
`126`	`126`	`"examples/run/caches/buildrunner.yaml",`
	`127`	`+ # This file is not supported in the github actions runner`
	`128`	`+ "examples/build/secrets/platforms-buildrunner.yaml",`
`127`	`129`	`]`
`128`	`130`
`129`	`131`	`# Walk through the examples directory and find all files ending with buildrunner.yaml`