migrate to packaging requirements parsing from deprecated pkg_resources (#22033)

Migrate to use `Requirement` from `packaging` instead of the deprecated
`pkg_resources`.

The `project_name` attribute is called `name` by
`packaging.requirements.Requirement` so migrate Pants naming to use
that, except in lock files where we keep the `project_name` naming to
avoid breaking existing lockfiles.

Note in the release notes that the `.*` pattern is only permitted with
`==` and `!=` operators. `pkg_resources` accepted `.*` patterns with
other operators which is not permitted by the current versioning
specification.

---------

Co-authored-by: Benjy Weinberger <[email protected]>

Author: tdyas

docs/notes/2.26.x.md

@@ -104,7 +104,9 @@ A bug in the Django backend has been fixed so that a repo may have no Django app
 
 [Pyright](https://www.pantsbuild.org/stable/reference/subsystems/pyright) was updated from version [1.1.383](https://github.com/microsoft/pyright/releases/tag/1.1.383) to [1.1.396](https://github.com/microsoft/pyright/releases/tag/1.1.396).
 
-The Python backend uses the PyPI `packaging` distribution to parse Python version numbers. In upgrading `packaging` from v21.3 to v24.2, support for parsing "legacy" Python versions was removed. We do not anticipate an issue here (since [PEP 440](https://peps.python.org/pep-0440/) has been around since 2013), but please contact the maintainers if an issue does arise.
+The Python backend uses the PyPI `packaging` distribution to parse Python version numbers and requirements instead of the deprecated `pkg_resources` distribution. In upgrading `packaging` from v21.3 to v24.2 and in removing uses of `pkg_resources`, several user-visible changes occurred:
+- There is no more support for parsing "legacy" Python versions. We do not anticipate an issue here (since [PEP 440](https://peps.python.org/pep-0440/) has been around since 2013), but please contact the maintainers if an issue does arise.
+- Requirements containing a version with a trailing `.*` pattern are only valid when used with the `==` or `!+` operators; for example, `==3.9.*` or `!=3.10.*`. Pants previously accepted such requirements when used with the other operators (e.g., `<=`, `<`, `>`, or `>=`) because `pkg_resources` accepted that syntax. The [current version specifiers specification](https://packaging.python.org/en/latest/specifications/version-specifiers/) does not allow that syntax and thus `packaging`'s requirements parser does not either.
 
 The Python Build Standalone backend (`pants.backend.python.providers.experimental.python_build_standalone`) has release metadata current through PBS release `20250212`.
 

src/python/pants/backend/openapi/codegen/python/generate.py

@@ -9,7 +9,7 @@
 from collections.abc import Iterable
 from dataclasses import dataclass
 
-from packaging.utils import canonicalize_name as canonicalize_project_name
+from packaging.utils import canonicalize_name
 
 from pants.backend.codegen.utils import MissingPythonCodegenRuntimeLibrary
 from pants.backend.openapi.codegen.python.extra_fields import (
@@ -256,9 +256,9 @@ async def get_python_requirements(
     result: defaultdict[str, dict[str, Address]] = defaultdict(dict)
     for target in python_targets.third_party:
         for python_requirement in target[PythonRequirementsField].value:
-            project_name = canonicalize_project_name(python_requirement.project_name)
+            name = canonicalize_name(python_requirement.name)
             resolve = target[PythonRequirementResolveField].normalized_value(python_setup)
-            result[resolve][project_name] = target.address
+            result[resolve][name] = target.address
 
     return PythonRequirements(
         resolves_to_requirements_to_addresses=FrozenDict(
@@ -313,16 +313,16 @@ async def infer_openapi_python_dependencies(
 
     addresses, missing_requirements = [], []
     for runtime_dependency in compiled_sources.runtime_dependencies:
-        project_name = runtime_dependency.project_name
-        address = requirements_to_addresses.get(project_name.lower())
+        name = canonicalize_name(runtime_dependency.name)
+        address = requirements_to_addresses.get(name)
         if address is not None:
             addresses.append(address)
         else:
-            missing_requirements.append(project_name)
+            missing_requirements.append(name)
 
     if missing_requirements:
         for_resolve_str = f" for the resolve '{resolve}'" if python_setup.enable_resolves else ""
-        missing = ", ".join(f"`{project_name}`" for project_name in missing_requirements)
+        missing = ", ".join(f"`{name}`" for name in missing_requirements)
         raise MissingPythonCodegenRuntimeLibrary(
             softwrap(
                 f"""

src/python/pants/backend/python/dependency_inference/module_mapper.py

@@ -375,8 +375,8 @@ def add_modules(modules: Iterable[str], *, is_type_stub: bool) -> None:
             # NB: We don't use `canonicalize_project_name()` for the fallback value because we
             # want to preserve `.` in the module name. See
             # https://www.python.org/dev/peps/pep-0503/#normalized-names.
-            proj_name = canonicalize_project_name(req.project_name)
-            fallback_value = req.project_name.strip().lower().replace("-", "_")
+            proj_name = canonicalize_project_name(req.name)
+            fallback_value = req.name.strip().lower().replace("-", "_")
 
             modules_to_add: tuple[str, ...]
             is_type_stub: bool

src/python/pants/backend/python/goals/pytest_runner.py

@@ -236,7 +236,7 @@ async def validate_pytest_cov_included(_pytest: PyTest):
         if not isinstance(lockfile_metadata, (PythonLockfileMetadataV2, PythonLockfileMetadataV3)):
             return
         requirements = lockfile_metadata.requirements
-    if not any(canonicalize_project_name(req.project_name) == "pytest-cov" for req in requirements):
+    if not any(canonicalize_project_name(req.name) == "pytest-cov" for req in requirements):
         raise ValueError(
             softwrap(
                 f"""\

src/python/pants/backend/python/goals/run_python_requirement.py

@@ -114,7 +114,7 @@ async def _resolve_entry_point(
         entry_point_module = module_mapping[field_set.address][0]
     elif len(reqs) == 1:
         # Use the canonicalized project name for a single-requirement target
-        entry_point_module = canonicalize_project_name(reqs[0].project_name)
+        entry_point_module = canonicalize_project_name(reqs[0].name)
     else:
         raise Exception(
             "Requirement must provide a single module, specify a single requirement, or specify "

src/python/pants/backend/python/macros/common_requirements_rule.py

@@ -151,9 +151,7 @@ def generate_tgt(
         )
 
     requirements = parse_requirements_callback(digest_contents[0].content, requirements_full_path)
-    grouped_requirements = itertools.groupby(
-        requirements, lambda parsed_req: parsed_req.project_name
-    )
+    grouped_requirements = itertools.groupby(requirements, lambda parsed_req: parsed_req.name)
     result = tuple(
         generate_tgt(project_name, parsed_reqs_)
         for project_name, parsed_reqs_ in grouped_requirements

src/python/pants/backend/python/subsystems/python_tool_base.py

@@ -242,11 +242,11 @@ def _default_package_name_and_version(cls) -> _PackageNameAndVersion | None:
         first_default_requirement = PipRequirement.parse(cls.default_requirements[0])
         return next(
             _PackageNameAndVersion(
-                name=first_default_requirement.project_name, version=requirement["version"]
+                name=first_default_requirement.name, version=requirement["version"]
             )
             for resolve in lockfile_contents["locked_resolves"]
             for requirement in resolve["locked_requirements"]
-            if requirement["project_name"] == first_default_requirement.project_name
+            if requirement["project_name"] == first_default_requirement.name
         )
 
     def pex_requirements(

src/python/pants/backend/python/util_rules/interpreter_constraints.py

@@ -10,8 +10,7 @@
 from collections.abc import Iterable, Iterator, Sequence
 from typing import Protocol, TypeVar
 
-from packaging.requirements import InvalidRequirement
-from pkg_resources import Requirement
+from packaging.requirements import InvalidRequirement, Requirement
 
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import InterpreterConstraintsField
@@ -67,13 +66,13 @@ def parse_constraint(constraint: str) -> Requirement:
     interpreter.py's `parse_requirement()`.
     """
     try:
-        parsed_requirement = Requirement.parse(constraint)
-    except ValueError as err:
+        parsed_requirement = Requirement(constraint)
+    except InvalidRequirement as err:
         try:
-            parsed_requirement = Requirement.parse(f"CPython{constraint}")
-        except ValueError:
+            parsed_requirement = Requirement(f"CPython{constraint}")
+        except InvalidRequirement:
             raise InvalidRequirement(
-                f"Failed to parse Python interpreter constraint `{constraint}`: {err.args[0]}"
+                f"Failed to parse Python interpreter constraint `{constraint}`: {err}"
             )
 
     return parsed_requirement
@@ -147,16 +146,15 @@ def merge_constraint_sets(
         if len(parsed_constraint_sets) == 1:
             return next(iter(parsed_constraint_sets))
 
-        def and_constraints(parsed_constraints: Sequence[Requirement]) -> Requirement:
-            merged_specs: set[tuple[str, str]] = set()
-            expected_interpreter = parsed_constraints[0].project_name
-            for parsed_constraint in parsed_constraints:
-                if parsed_constraint.project_name != expected_interpreter:
+        def and_constraints(parsed_requirements: Sequence[Requirement]) -> Requirement:
+            assert len(parsed_requirements) > 0, "At least one `Requirement` must be supplied."
+            expected_name = parsed_requirements[0].name
+            current_requirement_specifier = parsed_requirements[0].specifier
+            for requirement in parsed_requirements[1:]:
+                if requirement.name != expected_name:
                     return impossible
-                merged_specs.update(parsed_constraint.specs)
-
-            formatted_specs = ",".join(f"{op}{version}" for op, version in merged_specs)
-            return parse_constraint(f"{expected_interpreter}{formatted_specs}")
+                current_requirement_specifier &= requirement.specifier
+            return Requirement(f"{expected_name}{current_requirement_specifier}")
 
         ored_constraints = (
             and_constraints(constraints_product)
@@ -238,7 +236,7 @@ def generate_pex_arg_list(self) -> list[str]:
     def _valid_patch_versions(self, major: int, minor: int) -> Iterator[int]:
         for p in range(0, _PATCH_VERSION_UPPER_BOUND + 1):
             for req in self:
-                if req.specifier.contains(f"{major}.{minor}.{p}"):  # type: ignore[attr-defined]
+                if req.specifier.contains(f"{major}.{minor}.{p}"):
                     yield p
 
     def _includes_version(self, major: int, minor: int) -> bool:
@@ -271,9 +269,9 @@ def snap_to_minimum(self, interpreter_universe: Iterable[str]) -> InterpreterCon
         for major, minor in sorted(_major_minor_to_int(s) for s in interpreter_universe):
             for p in range(0, _PATCH_VERSION_UPPER_BOUND + 1):
                 for req in self:
-                    if req.specifier.contains(f"{major}.{minor}.{p}"):  # type: ignore[attr-defined]
+                    if req.specifier.contains(f"{major}.{minor}.{p}"):
                         # We've found the minimum major.minor that is compatible.
-                        req_strs = [f"{req.project_name}=={major}.{minor}.*"]
+                        req_strs = [f"{req.name}=={major}.{minor}.*"]
                         # Now find any patches within that major.minor that we must exclude.
                         invalid_patches = sorted(
                             set(range(0, _PATCH_VERSION_UPPER_BOUND + 1))
@@ -302,15 +300,9 @@ def _requires_python3_version_or_newer(
         ]
 
         def valid_constraint(constraint: Requirement) -> bool:
-            if any(
-                constraint.specifier.contains(prior)  # type: ignore[attr-defined]
-                for prior in prior_versions
-            ):
+            if any(constraint.specifier.contains(prior) for prior in prior_versions):
                 return False
-            if not any(
-                constraint.specifier.contains(allowed)  # type: ignore[attr-defined]
-                for allowed in allowed_versions
-            ):
+            if not any(constraint.specifier.contains(allowed) for allowed in allowed_versions):
                 return False
             return True
 
@@ -333,7 +325,7 @@ def to_poetry_constraint(self) -> str:
         specifiers = []
         wildcard_encountered = False
         for constraint in self:
-            specifier = str(constraint.specifier)  # type: ignore[attr-defined]
+            specifier = str(constraint.specifier)
             if specifier:
                 specifiers.append(specifier)
             else:
@@ -482,21 +474,23 @@ def _major_minor_version_when_single_and_entire(ics: InterpreterConstraints) ->
 
     req = next(iter(ics))
 
-    just_cpython = req.project_name == "CPython" and not req.extras and not req.marker
+    just_cpython = req.name == "CPython" and not req.extras and not req.marker
     if not just_cpython:
         raise _NonSimpleMajorMinor()
 
     # ==major.minor or ==major.minor.*
-    if len(req.specs) == 1:
-        operator, version = next(iter(req.specs))
-        if operator != "==":
+    if len(req.specifier) == 1:
+        specifier = next(iter(req.specifier))
+        if specifier.operator != "==":
             raise _NonSimpleMajorMinor()
 
-        return _parse_simple_version(version, require_any_patch=True)
+        return _parse_simple_version(specifier.version, require_any_patch=True)
 
     # >=major.minor,<major.(minor+1)
-    if len(req.specs) == 2:
-        (operator_lo, version_lo), (operator_hi, version_hi) = iter(req.specs)
+    if len(req.specifier) == 2:
+        specifiers = sorted(req.specifier, key=lambda s: s.version)
+        operator_lo, version_lo = (specifiers[0].operator, specifiers[0].version)
+        operator_hi, version_hi = (specifiers[1].operator, specifiers[1].version)
 
         if operator_lo != ">=":
             # if the lo operator isn't >=, they might be in the wrong order (or, if not, the check

src/python/pants/backend/python/util_rules/interpreter_constraints_test.py

@@ -7,8 +7,7 @@
 from dataclasses import dataclass
 
 import pytest
-from packaging.requirements import InvalidRequirement
-from pkg_resources import Requirement
+from packaging.requirements import InvalidRequirement, Requirement
 
 from pants.backend.python.subsystems.setup import PythonSetup
 from pants.backend.python.target_types import InterpreterConstraintsField
@@ -44,7 +43,7 @@ def assert_merged(*, inp: list[list[str]], expected: list[str]) -> None:
         result = sorted(str(req) for req in InterpreterConstraints.merge_constraint_sets(inp))
         # Requirement.parse() sorts specs differently than we'd like, so we convert each str to a
         # Requirement.
-        normalized_expected = sorted(str(Requirement.parse(v)) for v in expected)
+        normalized_expected = sorted(str(Requirement(v)) for v in expected)
         assert result == normalized_expected
 
     # Multiple constraint sets get merged so that they are ANDed.
@@ -324,7 +323,7 @@ def test_group_field_sets_by_constraints_with_unsorted_inputs() -> None:
         ),
     ]
 
-    ic_36 = InterpreterConstraints([Requirement.parse("CPython==3.6.*")])
+    ic_36 = InterpreterConstraints([Requirement("CPython==3.6.*")])
 
     output = InterpreterConstraints.group_field_sets_by_constraints(
         py3_fs,
@@ -490,7 +489,6 @@ def test_partition_into_major_minor_versions(constraints: list[str], expected: l
         (["==3.0.*"], (3, 0)),
         (["==3.45.*"], (3, 45)),
         ([">=3.45,<3.46"], (3, 45)),
-        ([">=3.45.*,<3.46.*"], (3, 45)),
         (["CPython>=3.45,<3.46"], (3, 45)),
         (["<3.46,>=3.45"], (3, 45)),
         # Invalid/too hard
@@ -514,7 +512,6 @@ def test_partition_into_major_minor_versions(constraints: list[str], expected: l
         ([">3.45,<=3.46"], None),
         ([">3.45,<3.47"], None),
         (["===3.45"], None),
-        ([">=3.45,<=3.45.*"], None),
         # wrong number of elements
         ([], None),
         (["==3.45.*", "==3.46.*"], None),
@@ -548,7 +545,7 @@ def test_major_minor_version_when_single_and_entire(
     ],
 )
 def test_parse_python_interpreter_constraint_when_valid(input_ic: str, expected: str) -> None:
-    assert parse_constraint(input_ic) == Requirement.parse(expected)
+    assert parse_constraint(input_ic) == Requirement(expected)
 
 
 @pytest.mark.parametrize(

src/python/pants/backend/python/util_rules/package_dists.py

@@ -651,7 +651,7 @@ async def determine_finalized_setup_kwargs(request: GenerateSetupPyRequest) -> F
             "python_requires",
             # Pick the first constraint using a generator detour, as the InterpreterConstraints is
             # based on a FrozenOrderedSet which is not indexable.
-            next(str(ic.specifier) for ic in request.interpreter_constraints),  # type: ignore[attr-defined]
+            next(str(ic.specifier) for ic in request.interpreter_constraints),
         )
 
     # The cascading defaults here are for two levels of "I know what I'm

src/python/pants/backend/python/util_rules/pex_from_targets.py

@@ -701,10 +701,10 @@ async def _setup_constraints_repository_pex(
             url_reqs.add(req)
         else:
             name_reqs.add(req)
-            name_req_projects.add(canonicalize_project_name(req.project_name))
+            name_req_projects.add(canonicalize_project_name(req.name))
 
     constraint_file_projects = {
-        canonicalize_project_name(req.project_name) for req in constraints_file_reqs
+        canonicalize_project_name(req.name) for req in constraints_file_reqs
     }
     # Constraints files must only contain name reqs, not URL reqs (those are already
     # constrained by their very nature). See https://github.com/pypa/pip/issues/8210.

src/python/pants/backend/python/util_rules/pex_test_utils.py

@@ -38,20 +38,20 @@ class ExactRequirement:
     @classmethod
     def parse(cls, requirement: str) -> ExactRequirement:
         req = PipRequirement.parse(requirement)
-        assert len(req.specs) == 1, softwrap(
+        assert len(req.specifier_set) == 1, softwrap(
             f"""
             Expected an exact requirement with only 1 specifier, given {requirement} with
-            {len(req.specs)} specifiers
+            {len(req.specifier_set)} specifiers
             """
         )
-        operator, version = req.specs[0]
-        assert operator == "==", softwrap(
+        specifier = next(iter(req.specifier_set))
+        assert specifier.operator == "==", softwrap(
             f"""
             Expected an exact requirement using only the '==' specifier, given {requirement}
-            using the {operator!r} operator
+            using the {specifier.operator!r} operator
             """
         )
-        return cls(project_name=req.project_name, version=version)
+        return cls(project_name=req.name, version=specifier.version)
 
 
 def parse_requirements(requirements: Iterable[str]) -> Iterator[ExactRequirement]: