diff --git a/docs/notes/2.33.x.md b/docs/notes/2.33.x.md index 1b63896348d..04d14d8ae68 100644 --- a/docs/notes/2.33.x.md +++ b/docs/notes/2.33.x.md @@ -24,6 +24,12 @@ The thread used by the [`workunit-logger`](https://www.pantsbuild.org/2.33/refer #### Docker +The option `[docker].build_extra_options` or/and the `build_extra_options` field of `docker_image` targets can now be used to pass extra options to docker build or podman build. Make sure that the options are understood by the tool. +For example, you can now set `build_extra_options=["--no-cache"]` to do not use existing cached images for the container build. + +Fields that are already present in the docker image target are surpressed by the extra options when they conflict. +For example, if you set `build_extra_options=["--pull"]`, the `pull` field of the docker image target will be ignored. + The `docker_image` target now supports capturing file and directory artifacts from Docker builds using the `output_files` and `output_directories` fields. This supports workflows where a Dockerfile stage creates a build artifact(s) that should be consumed by other Pants targets, using the same behavior as the `shell_command` or `adhoc_tool` targets. This feature uses the BuildKit local output exporter and requires `[docker].use_buildx = true`. #### Helm diff --git a/src/python/pants/backend/docker/goals/package_image.py b/src/python/pants/backend/docker/goals/package_image.py index 3dabc3455ea..f99acb5c836 100644 --- a/src/python/pants/backend/docker/goals/package_image.py +++ b/src/python/pants/backend/docker/goals/package_image.py @@ -27,6 +27,7 @@ DockerBuildOptionFieldMultiValueMixin, DockerBuildOptionFieldValueMixin, DockerBuildOptionFlagFieldMixin, + DockerImageBuildExtraOptionsField, DockerImageBuildImageOutputField, DockerImageContextRootField, DockerImageOutputDirectoriesField, @@ -346,6 +347,26 @@ class DockerInfoV1ImageTag: name: str +def _extra_options_flag_names(extra_options: tuple[str, ...]) -> frozenset[str]: + """Returns a set of flag names (e.g. --pull, -network, etc)""" + names: set[str] = set() + for opt in extra_options: + if opt.startswith("-"): + names.add(opt.split("=")[0]) + return frozenset(names) + + +def _filter_global_extra_options( + global_extra_options: tuple[str, ...], target_flag_names: frozenset[str] +) -> tuple[str, ...]: + """Remove any global extra options that are included in the per-target options.""" + return tuple( + opt + for opt in global_extra_options + if opt.startswith("-") and opt.split("=")[0] not in target_flag_names + ) + + def get_build_options( context: DockerBuildContext, field_set: DockerPackageFieldSet, @@ -354,8 +375,20 @@ def get_build_options( global_build_no_cache_option: bool | None, use_buildx_option: bool, target: Target, + global_extra_options: tuple[str, ...] = (), output_options: FrozenDict[str, str] | None = None, ) -> Iterator[str]: + target_extra = tuple(target[DockerImageBuildExtraOptionsField].value or ()) + target_flag_names = _extra_options_flag_names(target_extra) + + # Per-target wins: drop any global entry whose flag is already covered by the per-target list. + filtered_global = _filter_global_extra_options(global_extra_options, target_flag_names) + + extra_options: tuple[str, ...] = (*filtered_global, *target_extra) + + # Compute the set of flag names that are provided by global and target extra options. + overridden_flags = _extra_options_flag_names(extra_options) + # Build options from target fields inheriting from DockerBuildOptionFieldMixin for field_type in target.field_types: if issubclass(field_type, DockerBuildKitOptionField): @@ -380,6 +413,12 @@ def get_build_options( DockerBuildOptionFlagFieldMixin, ), ): + flag = getattr( + field_type, "docker_build_option", None + ) # get the flag name if it exists such as --pull or --network, etc. + if flag and flag in overridden_flags: + continue + source = InterpolationContext.TextSource( address=target.address, target_alias=target.alias, field_alias=field_type.alias ) @@ -416,11 +455,13 @@ def get_build_options( ) ) - if target_stage: - yield from ("--target", target_stage) + if target_stage and "--target" not in overridden_flags: + extra_options = extra_options + ("--target", target_stage) + + if global_build_no_cache_option and "--no-cache" not in overridden_flags: + extra_options = extra_options + ("--no-cache",) - if global_build_no_cache_option: - yield "--no-cache" + yield from extra_options @dataclass(frozen=True) @@ -619,6 +660,7 @@ async def get_docker_image_build_process( global_build_no_cache_option=options.build_no_cache, use_buildx_option=options.use_buildx, target=wrapped_target.target, + global_extra_options=options.build_extra_options, output_options=captured_outputs.output_options if captured_outputs else None, ) ), diff --git a/src/python/pants/backend/docker/goals/package_image_test.py b/src/python/pants/backend/docker/goals/package_image_test.py index 1af37d7d230..7b781dc08d5 100644 --- a/src/python/pants/backend/docker/goals/package_image_test.py +++ b/src/python/pants/backend/docker/goals/package_image_test.py @@ -156,6 +156,7 @@ def _setup_docker_options(rule_runner: RuleRunner, options: dict | None) -> Dock opts.setdefault("build_hosts", None) opts.setdefault("build_verbose", False) opts.setdefault("build_no_cache", False) + opts.setdefault("build_extra_options", []) opts.setdefault("use_buildx", False) opts.setdefault("env_vars", []) opts.setdefault("suggest_renames", True) @@ -3316,3 +3317,84 @@ def test_field_set_pushes_on_package(output: dict | None, expected: bool) -> Non rule_runner.get_target(Address("", target_name="image")) ) assert field_set.pushes_on_package() is expected + + +@pytest.mark.parametrize( + ("global_options", "target_options", "expected"), + [ + (["--compress"], None, ["--compress"]), + (["--compress"], ["--pull=always"], ["--compress", "--pull=always"]), + # The `--no-cache` option is set in a different place in the code and hence requires separate tests. + (["--no-cache"], None, ["--no-cache"]), + (None, ["--no-cache"], ["--no-cache"]), + ], +) +def test_global_build_extra_options_merge( + rule_runner: RuleRunner, + global_options: list | None, + target_options: list | None, + expected: list, +) -> None: + if global_options: + rule_runner.set_options( + [], + env={ + "PANTS_DOCKER_BUILD_EXTRA_OPTIONS": f"{global_options}", + }, + ) + rule_runner.write_files( + { + "docker/test/BUILD": dedent( + f"""\ + docker_image( + name="img1", + extra_build_options={target_options}, + ) + """ + ), + } + ) + + def check_build_process(result: DockerImageBuildProcess): + for expected_option in expected: + assert expected_option in result.process.argv + + assert_build_process( + rule_runner, + Address("docker/test", target_name="img1"), + build_process_assertions=check_build_process, + ) + + +def test_extra_build_options_overrides_pull_field_and_global_field(rule_runner: RuleRunner) -> None: + rule_runner.set_options( + [], + env={ + "PANTS_DOCKER_BUILD_EXTRA_OPTIONS": '["--pull=missing"]', + }, + ) + rule_runner.write_files( + { + "docker/test/BUILD": dedent( + """\ + docker_image( + name="img1", + pull=True, + extra_build_options=["--pull=always"], + ) + """ + ), + } + ) + + def check_build_process(result: DockerImageBuildProcess): + argv = result.process.argv + assert "--pull=always" in argv + assert "--pull=True" not in argv + assert "--pull=missing" not in argv + + assert_build_process( + rule_runner, + Address("docker/test", target_name="img1"), + build_process_assertions=check_build_process, + ) diff --git a/src/python/pants/backend/docker/subsystems/docker_options.py b/src/python/pants/backend/docker/subsystems/docker_options.py index 1787b3a0adf..7d1342bb967 100644 --- a/src/python/pants/backend/docker/subsystems/docker_options.py +++ b/src/python/pants/backend/docker/subsystems/docker_options.py @@ -207,6 +207,24 @@ def env_vars(self) -> tuple[str, ...]: default=False, help="Do not use the Docker cache when building images.", ) + _build_extra_options = ShellStrListOption( + help=softwrap( + f""" + Global extra options to pass to the `docker build` / `podman build` command. + + These are appended after all other computed flags. If any option specified here uses a + flag that is already covered by a dedicated field (e.g. `--pull`, `--network`, + `--platform`), the dedicated field's contribution is suppressed in favour of + the value given here, to avoid duplicate or conflicting flags. + + Example: + + [{options_scope}] + build_extra_options = ["--pull=always"] + + """ + ), + ) build_verbose = BoolOption( default=False, help="Whether to log the Docker output to the console. If false, only the image ID is logged.", @@ -302,6 +320,10 @@ def env_vars(self) -> tuple[str, ...]: def build_args(self) -> tuple[str, ...]: return tuple(sorted(set(self._build_args))) + @property + def build_extra_options(self) -> tuple[str, ...]: + return tuple(self._build_extra_options) + @property def tools(self) -> tuple[str, ...]: return tuple(sorted(set(self._tools))) diff --git a/src/python/pants/backend/docker/target_types.py b/src/python/pants/backend/docker/target_types.py index d2ebc4ab926..0da3bc103df 100644 --- a/src/python/pants/backend/docker/target_types.py +++ b/src/python/pants/backend/docker/target_types.py @@ -690,6 +690,30 @@ class DockerImageRunExtraArgsField(StringSequenceField): ) +class DockerImageBuildExtraOptionsField(StringSequenceField): + alias = "extra_build_options" + default = () + help = help_text( + lambda: f""" + Additional options to pass directly to the `docker build` (or `podman build`) command. + + These are appended after all other computed flags. If any option specified here uses a + flag that is already covered by a dedicated field (e.g. `--pull`, `--network`, + `--platform`), the dedicated field's contribution is suppressed in favour of + the value given here, to avoid duplicate or conflicting flags on the command line. + + Example: + + docker_image( + extra_build_options=["--pull=always", "--compress"], + ) + + Use `[{DockerOptions.options_scope}].build_extra_options` to set default options for all + images. Per-target option overrides a conflicting global one. + """ + ) + + class DockerImageTarget(Target): alias = "docker_image" core_fields = ( @@ -719,6 +743,7 @@ class DockerImageTarget(Target): DockerImageOutputDirectoriesField, DockerImageOutputsMatchModeField, DockerImageRunExtraArgsField, + DockerImageBuildExtraOptionsField, OutputPathField, RestartableField, )