diff --git a/docs/notes/2.26.x.md b/docs/notes/2.26.x.md index c745aafc809..f1921df8907 100644 --- a/docs/notes/2.26.x.md +++ b/docs/notes/2.26.x.md @@ -104,7 +104,9 @@ A bug in the Django backend has been fixed so that a repo may have no Django app [Pyright](https://www.pantsbuild.org/stable/reference/subsystems/pyright) was updated from version [1.1.383](https://github.com/microsoft/pyright/releases/tag/1.1.383) to [1.1.396](https://github.com/microsoft/pyright/releases/tag/1.1.396). -The Python backend uses the PyPI `packaging` distribution to parse Python version numbers. In upgrading `packaging` from v21.3 to v24.2, support for parsing "legacy" Python versions was removed. We do not anticipate an issue here (since [PEP 440](https://peps.python.org/pep-0440/) has been around since 2013), but please contact the maintainers if an issue does arise. +The Python backend uses the PyPI `packaging` distribution to parse Python version numbers and requirements instead of the deprecated `pkg_resources` distribution. In upgrading `packaging` from v21.3 to v24.2 and in removing uses of `pkg_resources`, several user-visible changes occurred: +- There is no more support for parsing "legacy" Python versions. We do not anticipate an issue here (since [PEP 440](https://peps.python.org/pep-0440/) has been around since 2013), but please contact the maintainers if an issue does arise. +- Requirements containing a version with a trailing `.*` pattern are only valid when used with the `==` or `!+` operators; for example, `==3.9.*` or `!=3.10.*`. Pants previously accepted such requirements when used with the other operators (e.g., `<=`, `<`, `>`, or `>=`) because `pkg_resources` accepted that syntax. The [current version specifiers specification](https://packaging.python.org/en/latest/specifications/version-specifiers/) does not allow that syntax and thus `packaging`'s requirements parser does not either. The Python Build Standalone backend (`pants.backend.python.providers.experimental.python_build_standalone`) has release metadata current through PBS release `20250212`. diff --git a/src/python/pants/backend/openapi/codegen/python/generate.py b/src/python/pants/backend/openapi/codegen/python/generate.py index d298b73a2b2..5a9bb005be6 100644 --- a/src/python/pants/backend/openapi/codegen/python/generate.py +++ b/src/python/pants/backend/openapi/codegen/python/generate.py @@ -9,7 +9,7 @@ from collections.abc import Iterable from dataclasses import dataclass -from packaging.utils import canonicalize_name as canonicalize_project_name +from packaging.utils import canonicalize_name from pants.backend.codegen.utils import MissingPythonCodegenRuntimeLibrary from pants.backend.openapi.codegen.python.extra_fields import ( @@ -256,9 +256,9 @@ async def get_python_requirements( result: defaultdict[str, dict[str, Address]] = defaultdict(dict) for target in python_targets.third_party: for python_requirement in target[PythonRequirementsField].value: - project_name = canonicalize_project_name(python_requirement.project_name) + name = canonicalize_name(python_requirement.name) resolve = target[PythonRequirementResolveField].normalized_value(python_setup) - result[resolve][project_name] = target.address + result[resolve][name] = target.address return PythonRequirements( resolves_to_requirements_to_addresses=FrozenDict( @@ -313,16 +313,16 @@ async def infer_openapi_python_dependencies( addresses, missing_requirements = [], [] for runtime_dependency in compiled_sources.runtime_dependencies: - project_name = runtime_dependency.project_name - address = requirements_to_addresses.get(project_name.lower()) + name = canonicalize_name(runtime_dependency.name) + address = requirements_to_addresses.get(name) if address is not None: addresses.append(address) else: - missing_requirements.append(project_name) + missing_requirements.append(name) if missing_requirements: for_resolve_str = f" for the resolve '{resolve}'" if python_setup.enable_resolves else "" - missing = ", ".join(f"`{project_name}`" for project_name in missing_requirements) + missing = ", ".join(f"`{name}`" for name in missing_requirements) raise MissingPythonCodegenRuntimeLibrary( softwrap( f""" diff --git a/src/python/pants/backend/python/dependency_inference/module_mapper.py b/src/python/pants/backend/python/dependency_inference/module_mapper.py index de5e4ffdc32..62dd41cb516 100644 --- a/src/python/pants/backend/python/dependency_inference/module_mapper.py +++ b/src/python/pants/backend/python/dependency_inference/module_mapper.py @@ -375,8 +375,8 @@ def add_modules(modules: Iterable[str], *, is_type_stub: bool) -> None: # NB: We don't use `canonicalize_project_name()` for the fallback value because we # want to preserve `.` in the module name. See # https://www.python.org/dev/peps/pep-0503/#normalized-names. - proj_name = canonicalize_project_name(req.project_name) - fallback_value = req.project_name.strip().lower().replace("-", "_") + proj_name = canonicalize_project_name(req.name) + fallback_value = req.name.strip().lower().replace("-", "_") modules_to_add: tuple[str, ...] is_type_stub: bool diff --git a/src/python/pants/backend/python/goals/pytest_runner.py b/src/python/pants/backend/python/goals/pytest_runner.py index b438d25e48d..8d3b7069550 100644 --- a/src/python/pants/backend/python/goals/pytest_runner.py +++ b/src/python/pants/backend/python/goals/pytest_runner.py @@ -236,7 +236,7 @@ async def validate_pytest_cov_included(_pytest: PyTest): if not isinstance(lockfile_metadata, (PythonLockfileMetadataV2, PythonLockfileMetadataV3)): return requirements = lockfile_metadata.requirements - if not any(canonicalize_project_name(req.project_name) == "pytest-cov" for req in requirements): + if not any(canonicalize_project_name(req.name) == "pytest-cov" for req in requirements): raise ValueError( softwrap( f"""\ diff --git a/src/python/pants/backend/python/goals/run_python_requirement.py b/src/python/pants/backend/python/goals/run_python_requirement.py index f0667f153b0..6772b85539a 100644 --- a/src/python/pants/backend/python/goals/run_python_requirement.py +++ b/src/python/pants/backend/python/goals/run_python_requirement.py @@ -114,7 +114,7 @@ async def _resolve_entry_point( entry_point_module = module_mapping[field_set.address][0] elif len(reqs) == 1: # Use the canonicalized project name for a single-requirement target - entry_point_module = canonicalize_project_name(reqs[0].project_name) + entry_point_module = canonicalize_project_name(reqs[0].name) else: raise Exception( "Requirement must provide a single module, specify a single requirement, or specify " diff --git a/src/python/pants/backend/python/macros/common_requirements_rule.py b/src/python/pants/backend/python/macros/common_requirements_rule.py index 6bf650841ee..a2b0ceadc26 100644 --- a/src/python/pants/backend/python/macros/common_requirements_rule.py +++ b/src/python/pants/backend/python/macros/common_requirements_rule.py @@ -151,9 +151,7 @@ def generate_tgt( ) requirements = parse_requirements_callback(digest_contents[0].content, requirements_full_path) - grouped_requirements = itertools.groupby( - requirements, lambda parsed_req: parsed_req.project_name - ) + grouped_requirements = itertools.groupby(requirements, lambda parsed_req: parsed_req.name) result = tuple( generate_tgt(project_name, parsed_reqs_) for project_name, parsed_reqs_ in grouped_requirements diff --git a/src/python/pants/backend/python/subsystems/python_tool_base.py b/src/python/pants/backend/python/subsystems/python_tool_base.py index 8651c221214..2851e08b14a 100644 --- a/src/python/pants/backend/python/subsystems/python_tool_base.py +++ b/src/python/pants/backend/python/subsystems/python_tool_base.py @@ -242,11 +242,11 @@ def _default_package_name_and_version(cls) -> _PackageNameAndVersion | None: first_default_requirement = PipRequirement.parse(cls.default_requirements[0]) return next( _PackageNameAndVersion( - name=first_default_requirement.project_name, version=requirement["version"] + name=first_default_requirement.name, version=requirement["version"] ) for resolve in lockfile_contents["locked_resolves"] for requirement in resolve["locked_requirements"] - if requirement["project_name"] == first_default_requirement.project_name + if requirement["project_name"] == first_default_requirement.name ) def pex_requirements( diff --git a/src/python/pants/backend/python/util_rules/interpreter_constraints.py b/src/python/pants/backend/python/util_rules/interpreter_constraints.py index a4406a68f4c..e8b5b847511 100644 --- a/src/python/pants/backend/python/util_rules/interpreter_constraints.py +++ b/src/python/pants/backend/python/util_rules/interpreter_constraints.py @@ -10,8 +10,7 @@ from collections.abc import Iterable, Iterator, Sequence from typing import Protocol, TypeVar -from packaging.requirements import InvalidRequirement -from pkg_resources import Requirement +from packaging.requirements import InvalidRequirement, Requirement from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import InterpreterConstraintsField @@ -67,13 +66,13 @@ def parse_constraint(constraint: str) -> Requirement: interpreter.py's `parse_requirement()`. """ try: - parsed_requirement = Requirement.parse(constraint) - except ValueError as err: + parsed_requirement = Requirement(constraint) + except InvalidRequirement as err: try: - parsed_requirement = Requirement.parse(f"CPython{constraint}") - except ValueError: + parsed_requirement = Requirement(f"CPython{constraint}") + except InvalidRequirement: raise InvalidRequirement( - f"Failed to parse Python interpreter constraint `{constraint}`: {err.args[0]}" + f"Failed to parse Python interpreter constraint `{constraint}`: {err}" ) return parsed_requirement @@ -147,16 +146,15 @@ def merge_constraint_sets( if len(parsed_constraint_sets) == 1: return next(iter(parsed_constraint_sets)) - def and_constraints(parsed_constraints: Sequence[Requirement]) -> Requirement: - merged_specs: set[tuple[str, str]] = set() - expected_interpreter = parsed_constraints[0].project_name - for parsed_constraint in parsed_constraints: - if parsed_constraint.project_name != expected_interpreter: + def and_constraints(parsed_requirements: Sequence[Requirement]) -> Requirement: + assert len(parsed_requirements) > 0, "At least one `Requirement` must be supplied." + expected_name = parsed_requirements[0].name + current_requirement_specifier = parsed_requirements[0].specifier + for requirement in parsed_requirements[1:]: + if requirement.name != expected_name: return impossible - merged_specs.update(parsed_constraint.specs) - - formatted_specs = ",".join(f"{op}{version}" for op, version in merged_specs) - return parse_constraint(f"{expected_interpreter}{formatted_specs}") + current_requirement_specifier &= requirement.specifier + return Requirement(f"{expected_name}{current_requirement_specifier}") ored_constraints = ( and_constraints(constraints_product) @@ -238,7 +236,7 @@ def generate_pex_arg_list(self) -> list[str]: def _valid_patch_versions(self, major: int, minor: int) -> Iterator[int]: for p in range(0, _PATCH_VERSION_UPPER_BOUND + 1): for req in self: - if req.specifier.contains(f"{major}.{minor}.{p}"): # type: ignore[attr-defined] + if req.specifier.contains(f"{major}.{minor}.{p}"): yield p def _includes_version(self, major: int, minor: int) -> bool: @@ -271,9 +269,9 @@ def snap_to_minimum(self, interpreter_universe: Iterable[str]) -> InterpreterCon for major, minor in sorted(_major_minor_to_int(s) for s in interpreter_universe): for p in range(0, _PATCH_VERSION_UPPER_BOUND + 1): for req in self: - if req.specifier.contains(f"{major}.{minor}.{p}"): # type: ignore[attr-defined] + if req.specifier.contains(f"{major}.{minor}.{p}"): # We've found the minimum major.minor that is compatible. - req_strs = [f"{req.project_name}=={major}.{minor}.*"] + req_strs = [f"{req.name}=={major}.{minor}.*"] # Now find any patches within that major.minor that we must exclude. invalid_patches = sorted( set(range(0, _PATCH_VERSION_UPPER_BOUND + 1)) @@ -302,15 +300,9 @@ def _requires_python3_version_or_newer( ] def valid_constraint(constraint: Requirement) -> bool: - if any( - constraint.specifier.contains(prior) # type: ignore[attr-defined] - for prior in prior_versions - ): + if any(constraint.specifier.contains(prior) for prior in prior_versions): return False - if not any( - constraint.specifier.contains(allowed) # type: ignore[attr-defined] - for allowed in allowed_versions - ): + if not any(constraint.specifier.contains(allowed) for allowed in allowed_versions): return False return True @@ -333,7 +325,7 @@ def to_poetry_constraint(self) -> str: specifiers = [] wildcard_encountered = False for constraint in self: - specifier = str(constraint.specifier) # type: ignore[attr-defined] + specifier = str(constraint.specifier) if specifier: specifiers.append(specifier) else: @@ -482,21 +474,23 @@ def _major_minor_version_when_single_and_entire(ics: InterpreterConstraints) -> req = next(iter(ics)) - just_cpython = req.project_name == "CPython" and not req.extras and not req.marker + just_cpython = req.name == "CPython" and not req.extras and not req.marker if not just_cpython: raise _NonSimpleMajorMinor() # ==major.minor or ==major.minor.* - if len(req.specs) == 1: - operator, version = next(iter(req.specs)) - if operator != "==": + if len(req.specifier) == 1: + specifier = next(iter(req.specifier)) + if specifier.operator != "==": raise _NonSimpleMajorMinor() - return _parse_simple_version(version, require_any_patch=True) + return _parse_simple_version(specifier.version, require_any_patch=True) # >=major.minor,=, they might be in the wrong order (or, if not, the check diff --git a/src/python/pants/backend/python/util_rules/interpreter_constraints_test.py b/src/python/pants/backend/python/util_rules/interpreter_constraints_test.py index 0fa8cf46628..6a85b641cf0 100644 --- a/src/python/pants/backend/python/util_rules/interpreter_constraints_test.py +++ b/src/python/pants/backend/python/util_rules/interpreter_constraints_test.py @@ -7,8 +7,7 @@ from dataclasses import dataclass import pytest -from packaging.requirements import InvalidRequirement -from pkg_resources import Requirement +from packaging.requirements import InvalidRequirement, Requirement from pants.backend.python.subsystems.setup import PythonSetup from pants.backend.python.target_types import InterpreterConstraintsField @@ -44,7 +43,7 @@ def assert_merged(*, inp: list[list[str]], expected: list[str]) -> None: result = sorted(str(req) for req in InterpreterConstraints.merge_constraint_sets(inp)) # Requirement.parse() sorts specs differently than we'd like, so we convert each str to a # Requirement. - normalized_expected = sorted(str(Requirement.parse(v)) for v in expected) + normalized_expected = sorted(str(Requirement(v)) for v in expected) assert result == normalized_expected # Multiple constraint sets get merged so that they are ANDed. @@ -324,7 +323,7 @@ def test_group_field_sets_by_constraints_with_unsorted_inputs() -> None: ), ] - ic_36 = InterpreterConstraints([Requirement.parse("CPython==3.6.*")]) + ic_36 = InterpreterConstraints([Requirement("CPython==3.6.*")]) output = InterpreterConstraints.group_field_sets_by_constraints( py3_fs, @@ -490,7 +489,6 @@ def test_partition_into_major_minor_versions(constraints: list[str], expected: l (["==3.0.*"], (3, 0)), (["==3.45.*"], (3, 45)), ([">=3.45,<3.46"], (3, 45)), - ([">=3.45.*,<3.46.*"], (3, 45)), (["CPython>=3.45,<3.46"], (3, 45)), (["<3.46,>=3.45"], (3, 45)), # Invalid/too hard @@ -514,7 +512,6 @@ def test_partition_into_major_minor_versions(constraints: list[str], expected: l ([">3.45,<=3.46"], None), ([">3.45,<3.47"], None), (["===3.45"], None), - ([">=3.45,<=3.45.*"], None), # wrong number of elements ([], None), (["==3.45.*", "==3.46.*"], None), @@ -548,7 +545,7 @@ def test_major_minor_version_when_single_and_entire( ], ) def test_parse_python_interpreter_constraint_when_valid(input_ic: str, expected: str) -> None: - assert parse_constraint(input_ic) == Requirement.parse(expected) + assert parse_constraint(input_ic) == Requirement(expected) @pytest.mark.parametrize( diff --git a/src/python/pants/backend/python/util_rules/package_dists.py b/src/python/pants/backend/python/util_rules/package_dists.py index a8482971805..103ecbef99a 100644 --- a/src/python/pants/backend/python/util_rules/package_dists.py +++ b/src/python/pants/backend/python/util_rules/package_dists.py @@ -651,7 +651,7 @@ async def determine_finalized_setup_kwargs(request: GenerateSetupPyRequest) -> F "python_requires", # Pick the first constraint using a generator detour, as the InterpreterConstraints is # based on a FrozenOrderedSet which is not indexable. - next(str(ic.specifier) for ic in request.interpreter_constraints), # type: ignore[attr-defined] + next(str(ic.specifier) for ic in request.interpreter_constraints), ) # The cascading defaults here are for two levels of "I know what I'm diff --git a/src/python/pants/backend/python/util_rules/pex_from_targets.py b/src/python/pants/backend/python/util_rules/pex_from_targets.py index 9f1ada986fa..96d32662c9c 100644 --- a/src/python/pants/backend/python/util_rules/pex_from_targets.py +++ b/src/python/pants/backend/python/util_rules/pex_from_targets.py @@ -701,10 +701,10 @@ async def _setup_constraints_repository_pex( url_reqs.add(req) else: name_reqs.add(req) - name_req_projects.add(canonicalize_project_name(req.project_name)) + name_req_projects.add(canonicalize_project_name(req.name)) constraint_file_projects = { - canonicalize_project_name(req.project_name) for req in constraints_file_reqs + canonicalize_project_name(req.name) for req in constraints_file_reqs } # Constraints files must only contain name reqs, not URL reqs (those are already # constrained by their very nature). See https://github.com/pypa/pip/issues/8210. diff --git a/src/python/pants/backend/python/util_rules/pex_test_utils.py b/src/python/pants/backend/python/util_rules/pex_test_utils.py index eb66ab4be6e..2f502f75972 100644 --- a/src/python/pants/backend/python/util_rules/pex_test_utils.py +++ b/src/python/pants/backend/python/util_rules/pex_test_utils.py @@ -38,20 +38,20 @@ class ExactRequirement: @classmethod def parse(cls, requirement: str) -> ExactRequirement: req = PipRequirement.parse(requirement) - assert len(req.specs) == 1, softwrap( + assert len(req.specifier_set) == 1, softwrap( f""" Expected an exact requirement with only 1 specifier, given {requirement} with - {len(req.specs)} specifiers + {len(req.specifier_set)} specifiers """ ) - operator, version = req.specs[0] - assert operator == "==", softwrap( + specifier = next(iter(req.specifier_set)) + assert specifier.operator == "==", softwrap( f""" Expected an exact requirement using only the '==' specifier, given {requirement} - using the {operator!r} operator + using the {specifier.operator!r} operator """ ) - return cls(project_name=req.project_name, version=version) + return cls(project_name=req.name, version=specifier.version) def parse_requirements(requirements: Iterable[str]) -> Iterator[ExactRequirement]: diff --git a/src/python/pants/util/pip_requirement.py b/src/python/pants/util/pip_requirement.py index 0fdd297d98c..d531b28cf9d 100644 --- a/src/python/pants/util/pip_requirement.py +++ b/src/python/pants/util/pip_requirement.py @@ -1,17 +1,13 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -# mypy: disable-error-code="import-untyped" -# Disables checking of the `pkg_resources` below which was awkward to figure out how to disable -# with a `# type: ignore[import-untyped]` comment. - from __future__ import annotations import logging import urllib.parse -import pkg_resources -from pkg_resources.extern.packaging.requirements import InvalidRequirement +from packaging.requirements import InvalidRequirement, Requirement +from packaging.specifiers import SpecifierSet logger = logging.getLogger(__name__) @@ -22,7 +18,7 @@ class PipRequirement: @classmethod def parse(cls, line: str, description_of_origin: str = "") -> PipRequirement: try: - return cls(pkg_resources.Requirement.parse(line)) + return cls(Requirement(line)) except InvalidRequirement as e: scheme, netloc, path, query, fragment = urllib.parse.urlsplit(line, scheme="file") if fragment: @@ -43,7 +39,7 @@ def parse(cls, line: str, description_of_origin: str = "") -> PipRequirement: full_url = urllib.parse.urlunsplit((scheme, netloc, path, query, fragment)) pep_440_req_str = f"{project}@ {full_url}" try: - return cls(pkg_resources.Requirement.parse(pep_440_req_str)) + return cls(Requirement(pep_440_req_str)) except InvalidRequirement: # If parsing the converted URL fails for some reason, it's probably less # confusing to the user if we raise the original error instead of one for @@ -52,19 +48,19 @@ def parse(cls, line: str, description_of_origin: str = "") -> PipRequirement: origin_str = f" in {description_of_origin}" if description_of_origin else "" raise ValueError(f"Invalid requirement '{line}'{origin_str}: {e}") - def __init__(self, req: pkg_resources.Requirement): + def __init__(self, req: Requirement): self._req = req - def as_pkg_resources_requirement(self) -> pkg_resources.Requirement: + def as_packaging_requirement(self) -> Requirement: return self._req @property - def project_name(self) -> str: - return self._req.project_name + def name(self) -> str: + return self._req.name @property - def specs(self): - return self._req.specs + def specifier_set(self) -> SpecifierSet: + return self._req.specifier @property def url(self): diff --git a/src/python/pants/util/pip_requirement_test.py b/src/python/pants/util/pip_requirement_test.py index 5193c7d11d7..628ad6398fe 100644 --- a/src/python/pants/util/pip_requirement_test.py +++ b/src/python/pants/util/pip_requirement_test.py @@ -1,14 +1,15 @@ # Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). import pytest +from packaging.specifiers import SpecifierSet from pants.util.pip_requirement import PipRequirement def test_parse_simple() -> None: req = PipRequirement.parse("Foo.bar==1.2.3") - assert req.project_name == "Foo.bar" - assert req.specs == [("==", "1.2.3")] + assert req.name == "Foo.bar" + assert req.specifier_set == SpecifierSet("==1.2.3") assert req.url is None @@ -22,15 +23,15 @@ def test_parse_simple() -> None: ) def test_parse_old_style_vcs(url: str, expected_project: str, expected_url: str) -> None: req = PipRequirement.parse(url) - assert req.project_name == expected_project - assert req.specs == [] + assert req.name == expected_project + assert req.specifier_set == SpecifierSet() assert req.url == expected_url or url def test_parse_pep440_vcs() -> None: req = PipRequirement.parse("Django@ git+https://github.com/django/django.git@stable/2.1.x") - assert req.project_name == "Django" - assert req.specs == [] + assert req.name == "Django" + assert req.specifier_set == SpecifierSet() assert req.url == "git+https://github.com/django/django.git@stable/2.1.x" diff --git a/src/python/pants/util/requirements.py b/src/python/pants/util/requirements.py index c31b41a281f..f91d36693ce 100644 --- a/src/python/pants/util/requirements.py +++ b/src/python/pants/util/requirements.py @@ -17,4 +17,9 @@ def parse_requirements_file(content: str, *, rel_path: str) -> Iterator[PipRequi if not line or line.startswith(("#", "-")): continue + # Strip comments which are otherwise on a valid requirement line. + comment_pos = line.find("#") + if comment_pos != -1: + line = line[0:comment_pos].strip() + yield PipRequirement.parse(line, description_of_origin=f"{rel_path} at line {i}")