diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 8dd21a1a6..014d501cd 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -50,7 +50,13 @@ from cibuildwheel.selector import BuildSelector, EnableGroup, TestSelector, selector_matches from cibuildwheel.typing import PLATFORMS, PlatformName from cibuildwheel.util import resources -from cibuildwheel.util.helpers import format_safe, parse_key_value_string, strtobool, unwrap +from cibuildwheel.util.helpers import ( + format_safe, + parse_key_value_string, + parse_kw_string, + strtobool, + unwrap, +) from cibuildwheel.util.packaging import DependencyConstraints TYPE_CHECKING = False @@ -436,6 +442,28 @@ def _stringify_setting( return setting +def parse_inherit(config: str | dict[str, str] | None) -> dict[str, InheritRule]: + inherit_dict: dict[str, str] + + if config is None: + return {} + + if isinstance(config, str): + parsed = parse_kw_string(config, default_kw_value="append") + inherit_dict = {k: "".join(v).upper() for k, v in parsed.items()} + elif isinstance(config, dict): + inherit_dict = config + else: + msg = "'inherit' must be a string or a table" + raise OptionsReaderError(msg) + + if not all(v in {"none", "append", "prepend"} for v in inherit_dict.values()): + msg = "'inherit' must contain only {'none', 'append', 'prepend'} values" + raise OptionsReaderError(msg) + + return {k: InheritRule[v.upper()] for k, v in inherit_dict.items()} + + class OptionsReader: """ Gets options from the environment, config or defaults, optionally scoped @@ -484,39 +512,12 @@ def __init__( self._validate_platform_option(option_name) self.config_options = config_options + self.config_options_inherit = parse_inherit(config_options.get("inherit")) self.config_platform_options = config_platform_options + self.config_platform_options_inherit = parse_inherit(config_platform_options.get("inherit")) - self.overrides: list[Override] = [] self.current_identifier: str | None = None - config_overrides = self.config_options.get("overrides") - - if config_overrides is not None: - if not isinstance(config_overrides, list): - msg = "'tool.cibuildwheel.overrides' must be a list" - raise OptionsReaderError(msg) - - for config_override in config_overrides: - select = config_override.pop("select", None) - - if not select: - msg = "'select' must be set in an override" - raise OptionsReaderError(msg) - - if isinstance(select, list): - select = " ".join(select) - - inherit = config_override.pop("inherit", {}) - if not isinstance(inherit, dict) or not all( - i in {"none", "append", "prepend"} for i in inherit.values() - ): - msg = "'inherit' must be a dict containing only {'none', 'append', 'prepend'} values" - raise OptionsReaderError(msg) - - inherit_enum = {k: InheritRule[v.upper()] for k, v in inherit.items()} - - self.overrides.append(Override(select, config_override, inherit_enum)) - def _validate_global_option(self, name: str) -> None: """ Raises an error if an option with this name is not allowed in the @@ -562,6 +563,56 @@ def _load_file(self, filename: Path) -> tuple[dict[str, Any], dict[str, Any]]: return global_options, platform_options + @functools.cached_property + def overrides(self) -> list[Override]: + config_overrides = self.config_options.get("overrides") + overrides: list[Override] = [] + + if config_overrides is not None: + if not isinstance(config_overrides, list): + msg = "'tool.cibuildwheel.overrides' must be a list" + raise OptionsReaderError(msg) + + for config_override in config_overrides: + select = config_override.pop("select", None) + + if not select: + msg = "'select' must be set in an override" + raise OptionsReaderError(msg) + + if isinstance(select, list): + select = " ".join(select) + + inherit = config_override.pop("inherit", {}) + + overrides.append(Override(select, config_override, parse_inherit(inherit))) + + return overrides + + @functools.cached_property + def env_inherit(self) -> dict[str, InheritRule]: + env_inherit_str = self.env.get("CIBW_INHERIT", "") + try: + return parse_inherit(env_inherit_str) + except OptionsReaderError as e: + msg = f"Failed to parse CIBW_INHERIT environment variable. {e}" + raise errors.ConfigurationError(msg) from e + + @functools.cached_property + def env_platform_inherit(self) -> dict[str, InheritRule]: + env_inherit = self.env_inherit + + # find the rules which have -{platform} on the end of their key, + # remove the platform suffix from the key and return the resulting + # rule. + platform_suffix = f"-{self.platform}" + + return { + key.removesuffix(platform_suffix): value + for key, value in env_inherit.items() + if key.endswith(platform_suffix) + } + @property def active_config_overrides(self) -> list[Override]: if self.current_identifier is None: @@ -585,7 +636,7 @@ def get( env_plat: bool = True, option_format: OptionFormat | None = None, ignore_empty: bool = False, - env_rule: InheritRule = InheritRule.NONE, + default_env_rule: InheritRule = InheritRule.NONE, ) -> str: """ Get and return the value for the named option from environment, @@ -609,16 +660,37 @@ def get( # get the option from the default, then the config file, then finally the environment. # platform-specific options are preferred, if they're allowed. return _resolve_cascade( - (self.default_options.get(name), InheritRule.NONE), - (self.default_platform_options.get(name), InheritRule.NONE), - (self.config_options.get(name), InheritRule.NONE), - (self.config_platform_options.get(name), InheritRule.NONE), + ( + self.default_options.get(name), + InheritRule.NONE, + ), + ( + self.default_platform_options.get(name), + InheritRule.NONE, + ), + ( + self.config_options.get(name), + self.config_options_inherit.get(name, InheritRule.NONE), + ), + ( + self.config_platform_options.get(name), + self.config_platform_options_inherit.get(name, InheritRule.NONE), + ), *[ - (o.options.get(name), o.inherit.get(name, InheritRule.NONE)) + ( + o.options.get(name), + o.inherit.get(name, InheritRule.NONE), + ) for o in self.active_config_overrides ], - (self.env.get(envvar), env_rule), - (self.env.get(plat_envvar) if env_plat else None, env_rule), + ( + self.env.get(envvar), + self.env_inherit.get(name, default_env_rule), + ), + ( + self.env.get(plat_envvar) if env_plat else None, + self.env_platform_inherit.get(name, default_env_rule), + ), ignore_empty=ignore_empty, option_format=option_format, ) @@ -691,7 +763,10 @@ def globals(self) -> GlobalOptions: allow_empty = args.allow_empty or strtobool(self.env.get("CIBW_ALLOW_EMPTY", "0")) enable_groups = self.reader.get( - "enable", env_plat=False, option_format=ListFormat(sep=" "), env_rule=InheritRule.APPEND + "enable", + env_plat=False, + option_format=ListFormat(sep=" "), + default_env_rule=InheritRule.APPEND, ) try: enable = { @@ -797,12 +872,11 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: if xbuild_tools == ["\u0000"]: xbuild_tools = None - xbuild_files = parse_key_value_string( + xbuild_files = parse_kw_string( self.reader.get( "xbuild-files", option_format=ShlexTableFormat(sep="; ", pair_sep=":", allow_merge=False), ), - kw_arg_names=["*"], ) test_sources = shlex.split( diff --git a/cibuildwheel/util/helpers.py b/cibuildwheel/util/helpers.py index d6ed17971..659df9356 100644 --- a/cibuildwheel/util/helpers.py +++ b/cibuildwheel/util/helpers.py @@ -111,7 +111,7 @@ def parse_key_value_string( if kw_arg_names is None: kw_arg_names = [] - all_field_names = None if ("*" in kw_arg_names) else [*positional_arg_names, *kw_arg_names] + all_field_names = [*positional_arg_names, *kw_arg_names] shlexer = shlex.shlex(key_value_string, posix=True, punctuation_chars=";") shlexer.commenters = "" @@ -128,7 +128,7 @@ def parse_key_value_string( # check to see if the option name is specified field_name, sep, first_value = field[0].partition(":") if sep: - if (all_field_names is not None) and (field_name not in all_field_names): + if field_name not in all_field_names: msg = f"Failed to parse {key_value_string!r}. Unknown field name {field_name!r}" raise ValueError(msg) @@ -147,6 +147,49 @@ def parse_key_value_string( return dict(result) +def parse_kw_string(kw_string: str, default_kw_value: str | None = None) -> dict[str, list[str]]: + """ + Parses a string like + + "before-build; before-test: append; after-test: prepend" + or + "package1: some/header.h some/library.a; package2: other/header.h" + + There are no restrictions on the keys than can be set. + + No positional arguments are allowed. Words without a colon attached are + interpreted as keys. Keys without a value will be assigned the + default_kw_value if provided, otherwise an empty list. + """ + shlexer = shlex.shlex(kw_string, posix=True, punctuation_chars=";") + shlexer.commenters = "" + shlexer.whitespace_split = True + parts = list(shlexer) + # parts now looks like + # ['before-build', ';', 'before-test:', 'append', ';', 'after-test:', 'prepend'] + + # split by semicolon + result: defaultdict[str, list[str]] = defaultdict(list) + fields = [list(group) for k, group in itertools.groupby(parts, lambda x: x == ";") if not k] + for field in fields: + # check to see if the option name is specified + field_name, sep, first_value = field[0].partition(":") + if sep: + # the colon was present, so the first value is the value after the colon + values = ([first_value] if first_value else []) + field[1:] + result[field_name] += values + else: + # no colon, so it's a key (or set of keys) without values + if default_kw_value is None: + msg = f"Failed to parse {kw_string!r}. No value specified for {field_name!r}. Expected ':' followed by a value." + raise ValueError(msg) + + for key in field: + result[key].append(default_kw_value) + + return dict(result) + + @dataclasses.dataclass(order=True) class FlexibleVersion: version_parts: tuple[int, ...] = dataclasses.field(init=False, repr=False) diff --git a/unit_test/options_toml_test.py b/unit_test/options_toml_test.py index d4008a2c4..dc14d3920 100644 --- a/unit_test/options_toml_test.py +++ b/unit_test/options_toml_test.py @@ -472,7 +472,11 @@ def test_overrides_not_a_list(tmp_path: Path, platform: str) -> None: ) with pytest.raises(OptionsReaderError): - OptionsReader(config_file_path=pyproject_toml, platform=cast("Any", platform), env={}) + print( + OptionsReader( + config_file_path=pyproject_toml, platform=cast("Any", platform), env={} + ).overrides + ) def test_config_settings(tmp_path: Path) -> None: diff --git a/unit_test/utils_test.py b/unit_test/utils_test.py index 2322a0579..189f42627 100644 --- a/unit_test/utils_test.py +++ b/unit_test/utils_test.py @@ -12,6 +12,7 @@ FlexibleVersion, format_safe, parse_key_value_string, + parse_kw_string, prepare_command, unwrap, unwrap_preserving_paragraphs, @@ -218,20 +219,57 @@ def test_parse_key_value_string_unknown_name() -> None: with pytest.raises(ValueError, match=r"Failed to parse 'key: value'. Unknown field name 'key'"): parse_key_value_string("key: value") - # Unknown fields can be enabled by passing "*". - assert parse_key_value_string( - "key: value", - kw_arg_names=["*"], - ) == { - "key": ["value"], + +def test_parse_kw_string_basic() -> None: + assert parse_kw_string("before-test: append; after-test: prepend") == { + "before-test": ["append"], + "after-test": ["prepend"], } - assert parse_key_value_string( - "key1: value1a value1b; key2: value2", - kw_arg_names=["*"], - ) == { - "key1": ["value1a", "value1b"], - "key2": ["value2"], + +def test_parse_kw_string_multiple_values() -> None: + assert parse_kw_string("package1: some/header.h some/library.a; package2: other/header.h") == { + "package1": ["some/header.h", "some/library.a"], + "package2": ["other/header.h"], + } + + +def test_parse_kw_string_keys_without_values_default() -> None: + assert parse_kw_string("before-build; before-test: prepend", default_kw_value="append") == { + "before-build": ["append"], + "before-test": ["prepend"], + } + + +def test_parse_kw_string_keys_without_values_no_default() -> None: + with pytest.raises(ValueError, match="No value specified"): + parse_kw_string("before-build") + + +def test_parse_kw_string_empty() -> None: + assert parse_kw_string("") == {} + + +def test_parse_kw_string_duplicate_keys() -> None: + assert parse_kw_string("key: val1; key: val2") == { + "key": ["val1", "val2"], + } + + +def test_parse_kw_string_key_only_with_colon() -> None: + assert parse_kw_string("key:") == {"key": []} + + +def test_parse_kw_string_quoted_values() -> None: + assert parse_kw_string('key: "hello world"') == {"key": ["hello world"]} + + +def test_parse_kw_string_multiple_bare_keys_with_default() -> None: + """works, but should remain undocumented""" + assert parse_kw_string("a b c", default_kw_value="yes") == { + "a": ["yes"], + "b": ["yes"], + "c": ["yes"], }