From 218853c96d781ccc0e365db29d99a91bc056a5bd Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Sat, 14 May 2022 12:53:51 +1000 Subject: [PATCH 1/5] . --- .../pip_install/extract_wheels/lib/bazel.py | 19 +++--- .../pip_install/extract_wheels/lib/wheel.py | 62 ++++++++----------- python/pip_install/repositories.bzl | 10 +-- 3 files changed, 39 insertions(+), 52 deletions(-) diff --git a/python/pip_install/extract_wheels/lib/bazel.py b/python/pip_install/extract_wheels/lib/bazel.py index ecb91fc828..c44af5f7c3 100644 --- a/python/pip_install/extract_wheels/lib/bazel.py +++ b/python/pip_install/extract_wheels/lib/bazel.py @@ -21,29 +21,28 @@ def generate_entry_point_contents( - entry_point: str, shebang: str = "#!/usr/bin/env python3" + module: str, attribute: str, shebang: str = "#!/usr/bin/env python3" ) -> str: """Generate the contents of an entry point script. Args: - entry_point (str): The name of the entry point as show in the - `console_scripts` section of `entry_point.txt`. + module (str): The name of the module to use. + attribute (str): The name of the attribute to call. shebang (str, optional): The shebang to use for the entry point python file. Returns: str: A string of python code. """ - module, method = entry_point.split(":", 1) return textwrap.dedent( """\ {shebang} import sys - from {module} import {method} + from {module} import {attribute} if __name__ == "__main__": - sys.exit({method}()) + sys.exit({attribute}()) """.format( - shebang=shebang, module=module, method=method + shebang=shebang, module=module, attribute=attribute ) ) @@ -408,10 +407,10 @@ def extract_wheel( directory_path = Path(directory) entry_points = [] - for name, entry_point in sorted(whl.entry_points().items()): + for name, module, attribute in sorted(whl.entry_points().items()): entry_point_script = f"{WHEEL_ENTRY_POINT_PREFIX}_{name}.py" (directory_path / entry_point_script).write_text( - generate_entry_point_contents(entry_point) + generate_entry_point_contents(module, attribute) ) entry_points.append( generate_entry_point_rule( @@ -449,7 +448,7 @@ def extract_wheel( data_exclude=data_exclude, data=data, srcs_exclude=srcs_exclude, - tags=["pypi_name=" + whl.name, "pypi_version=" + whl.metadata.version], + tags=["pypi_name=" + whl.name, "pypi_version=" + whl.version], additional_content=additional_content, ) build_file.write(contents) diff --git a/python/pip_install/extract_wheels/lib/wheel.py b/python/pip_install/extract_wheels/lib/wheel.py index 85bc95830d..70f4c2e6ac 100644 --- a/python/pip_install/extract_wheels/lib/wheel.py +++ b/python/pip_install/extract_wheels/lib/wheel.py @@ -1,13 +1,14 @@ """Utility class to inspect an extracted wheel directory""" import configparser +import email import glob import os import stat import zipfile from typing import Dict, Optional, Set +import installer import pkg_resources -import pkginfo def current_umask() -> int: @@ -37,57 +38,44 @@ def path(self) -> str: @property def name(self) -> str: - return str(self.metadata.name) + # TODO Also available as installer.sources.WheelSource.distribution + return str(self.metadata['Name']) @property - def metadata(self) -> pkginfo.Wheel: - return pkginfo.get_metadata(self.path) + def metadata(self) -> email.message.Message: + with WheelFile.open(self.path) as wheel_source: + metadata_file = wheel_source.read_dist_info("METADATA") + metadata = installer.utils.parse_metadata_file(metadata_file) + return metadata - def entry_points(self) -> Dict[str, str]: + @property + def version(self) -> str: + # TODO Also available as installer.sources.WheelSource.version + return str(self.metadata["Version"]) + + def entry_points(self) -> Dict[str, Tuple[str, str]]: """Returns the entrypoints defined in the current wheel See https://packaging.python.org/specifications/entry-points/ for more info Returns: - Dict[str, str]: A mappying of the entry point's name to it's method + Dict[str, Tuple[str, str]]: A mapping of the entry point's name to it's module and attribute """ - with zipfile.ZipFile(self.path, "r") as whl: - # Calculate the location of the entry_points.txt file - metadata = self.metadata - name = "{}-{}".format(metadata.name.replace("-", "_"), metadata.version) - - # Note that the zipfile module always uses the forward slash as - # directory separator, even on Windows, so don't use os.path.join - # here. Reference for Python 3.10: - # https://github.com/python/cpython/blob/3.10/Lib/zipfile.py#L355. - # TODO: use zipfile.Path once 3.8 is our minimum supported version - entry_points_path = "{}.dist-info/entry_points.txt".format(name) - - # If this file does not exist in the wheel, there are no entry points - if entry_points_path not in whl.namelist(): + with WheelFile.open(self.path) as wheel_source: + entry_points_file = wheel_source.read_dist_info("entry_points.txt") + if entry_points_file is None: return dict() - # Parse the avaialble entry points - config = configparser.ConfigParser() - try: - config.read_string(whl.read(entry_points_path).decode("utf-8")) - if "console_scripts" in config.sections(): - return dict(config["console_scripts"]) - - # TODO: It's unclear what to do in a situation with duplicate sections or options. - # For now, we treat the config file as though it contains no scripts. For more - # details on the config parser, see: - # https://docs.python.org/3.7/library/configparser.html#configparser.ConfigParser - # https://docs.python.org/3.7/library/configparser.html#configparser.Error - except configparser.Error: - pass - - return dict() + d = dict() + entry_points = installer.utils.parse_entrypoints(entry_points_file) + for script, module, attribute, kind in entry_points: + if kind == "console": + d[script] = (module, attribute) def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]: dependency_set = set() - for wheel_req in self.metadata.requires_dist: + for wheel_req in self.metadata.get_all('Requires-Dist'): req = pkg_resources.Requirement(wheel_req) # type: ignore if req.marker is None or any( diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl index 352f66341c..a57ff9deaa 100644 --- a/python/pip_install/repositories.bzl +++ b/python/pip_install/repositories.bzl @@ -17,6 +17,11 @@ _RULE_DEPS = [ "https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl", "9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2", ), + ( + "pypi__installer", + "https://files.pythonhosted.org/packages/1b/21/3e6ebd12d8dccc55bcb7338db462c75ac86dbd0ac7439ac114616b21667b/installer-0.5.1-py3-none-any.whl", + "1d6c8d916ed82771945b9c813699e6f57424ded970c9d8bf16bbc23e1e826ed3", + ), ( "pypi__pip", "https://files.pythonhosted.org/packages/4d/16/0a14ca596f30316efd412a60bdfac02a7259bf8673d4d917dc60b9a21812/pip-22.0.4-py3-none-any.whl", @@ -27,11 +32,6 @@ _RULE_DEPS = [ "https://files.pythonhosted.org/packages/6d/16/75d65bdccd48bb59a08e2bf167b01d8532f65604270d0a292f0f16b7b022/pip_tools-5.5.0-py2.py3-none-any.whl", "10841c1e56c234d610d0466447685b9ea4ee4a2c274f858c0ef3c33d9bd0d985", ), - ( - "pypi__pkginfo", - "https://files.pythonhosted.org/packages/cd/00/49f59cdd2c6a52e6665fda4de671dac5614366dc827e050c55428241b929/pkginfo-1.8.2-py2.py3-none-any.whl", - "c24c487c6a7f72c66e816ab1796b96ac6c3d14d49338293d2141664330b55ffc", - ), ( "pypi__setuptools", "https://files.pythonhosted.org/packages/7c/5b/3d92b9f0f7ca1645cba48c080b54fe7d8b1033a4e5720091d1631c4266db/setuptools-60.10.0-py3-none-any.whl", From b4acf79f796e803b3637d8b7a6e83be1b9b4b45e Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Sat, 14 May 2022 13:35:38 +1000 Subject: [PATCH 2/5] . --- python/pip_install/extract_wheels/lib/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pip_install/extract_wheels/lib/BUILD b/python/pip_install/extract_wheels/lib/BUILD index 3e2f307a73..48214126e4 100644 --- a/python/pip_install/extract_wheels/lib/BUILD +++ b/python/pip_install/extract_wheels/lib/BUILD @@ -18,7 +18,7 @@ py_library( "//python/pip_install/parse_requirements_to_bzl:__subpackages__", ], deps = [ - requirement("pkginfo"), + requirement("installer"), requirement("setuptools"), ], ) From 5ebb7ae8ae3168a039ee3e286ff7373b74e53b65 Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Sat, 14 May 2022 13:43:11 +1000 Subject: [PATCH 3/5] . --- python/pip_install/extract_wheels/lib/wheel.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/python/pip_install/extract_wheels/lib/wheel.py b/python/pip_install/extract_wheels/lib/wheel.py index 70f4c2e6ac..e5912a6f79 100644 --- a/python/pip_install/extract_wheels/lib/wheel.py +++ b/python/pip_install/extract_wheels/lib/wheel.py @@ -1,5 +1,4 @@ """Utility class to inspect an extracted wheel directory""" -import configparser import email import glob import os @@ -43,7 +42,7 @@ def name(self) -> str: @property def metadata(self) -> email.message.Message: - with WheelFile.open(self.path) as wheel_source: + with installer.sources.WheelFile.open(self.path) as wheel_source: metadata_file = wheel_source.read_dist_info("METADATA") metadata = installer.utils.parse_metadata_file(metadata_file) return metadata @@ -53,7 +52,7 @@ def version(self) -> str: # TODO Also available as installer.sources.WheelSource.version return str(self.metadata["Version"]) - def entry_points(self) -> Dict[str, Tuple[str, str]]: + def entry_points(self) -> Dict[str, tuple[str, str]]: """Returns the entrypoints defined in the current wheel See https://packaging.python.org/specifications/entry-points/ for more info @@ -61,7 +60,7 @@ def entry_points(self) -> Dict[str, Tuple[str, str]]: Returns: Dict[str, Tuple[str, str]]: A mapping of the entry point's name to it's module and attribute """ - with WheelFile.open(self.path) as wheel_source: + with installer.sources.WheelFile.open(self.path) as wheel_source: entry_points_file = wheel_source.read_dist_info("entry_points.txt") if entry_points_file is None: return dict() @@ -75,7 +74,7 @@ def entry_points(self) -> Dict[str, Tuple[str, str]]: def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]: dependency_set = set() - for wheel_req in self.metadata.get_all('Requires-Dist'): + for wheel_req in self.metadata.get_all('Requires-Dist', list()): req = pkg_resources.Requirement(wheel_req) # type: ignore if req.marker is None or any( From 71bb5d0509ad2450b94102da931c1ac197e81340 Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Sat, 14 May 2022 13:55:42 +1000 Subject: [PATCH 4/5] . --- python/pip_install/extract_wheels/lib/bazel.py | 2 +- python/pip_install/extract_wheels/lib/wheel.py | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/python/pip_install/extract_wheels/lib/bazel.py b/python/pip_install/extract_wheels/lib/bazel.py index c44af5f7c3..9e181a4109 100644 --- a/python/pip_install/extract_wheels/lib/bazel.py +++ b/python/pip_install/extract_wheels/lib/bazel.py @@ -407,7 +407,7 @@ def extract_wheel( directory_path = Path(directory) entry_points = [] - for name, module, attribute in sorted(whl.entry_points().items()): + for name, (module, attribute) in sorted(whl.entry_points().items()): entry_point_script = f"{WHEEL_ENTRY_POINT_PREFIX}_{name}.py" (directory_path / entry_point_script).write_text( generate_entry_point_contents(module, attribute) diff --git a/python/pip_install/extract_wheels/lib/wheel.py b/python/pip_install/extract_wheels/lib/wheel.py index e5912a6f79..3d4ebd1c0d 100644 --- a/python/pip_install/extract_wheels/lib/wheel.py +++ b/python/pip_install/extract_wheels/lib/wheel.py @@ -43,8 +43,8 @@ def name(self) -> str: @property def metadata(self) -> email.message.Message: with installer.sources.WheelFile.open(self.path) as wheel_source: - metadata_file = wheel_source.read_dist_info("METADATA") - metadata = installer.utils.parse_metadata_file(metadata_file) + metadata_contents = wheel_source.read_dist_info("METADATA") + metadata = installer.utils.parse_metadata_file(metadata_contents) return metadata @property @@ -61,15 +61,17 @@ def entry_points(self) -> Dict[str, tuple[str, str]]: Dict[str, Tuple[str, str]]: A mapping of the entry point's name to it's module and attribute """ with installer.sources.WheelFile.open(self.path) as wheel_source: - entry_points_file = wheel_source.read_dist_info("entry_points.txt") - if entry_points_file is None: + if "entry_points.txt" not in wheel_source.dist_info_filenames: return dict() - d = dict() - entry_points = installer.utils.parse_entrypoints(entry_points_file) + entry_points_mapping = dict() + entry_points_contents = wheel_source.read_dist_info("entry_points.txt") + entry_points = installer.utils.parse_entrypoints(entry_points_contents) for script, module, attribute, kind in entry_points: if kind == "console": - d[script] = (module, attribute) + entry_points_mapping[script] = (module, attribute) + + return entry_points_mapping def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]: dependency_set = set() From 6d68886218fe4919bdc861bb09b4c974e34bc5bd Mon Sep 17 00:00:00 2001 From: Greg Roodt Date: Fri, 20 May 2022 19:34:39 +1000 Subject: [PATCH 5/5] Fixes from review. --- python/pip_install/extract_wheels/lib/wheel.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pip_install/extract_wheels/lib/wheel.py b/python/pip_install/extract_wheels/lib/wheel.py index 3d4ebd1c0d..73d5eb53da 100644 --- a/python/pip_install/extract_wheels/lib/wheel.py +++ b/python/pip_install/extract_wheels/lib/wheel.py @@ -67,8 +67,8 @@ def entry_points(self) -> Dict[str, tuple[str, str]]: entry_points_mapping = dict() entry_points_contents = wheel_source.read_dist_info("entry_points.txt") entry_points = installer.utils.parse_entrypoints(entry_points_contents) - for script, module, attribute, kind in entry_points: - if kind == "console": + for script, module, attribute, script_section in entry_points: + if script_section == "console": entry_points_mapping[script] = (module, attribute) return entry_points_mapping @@ -76,7 +76,7 @@ def entry_points(self) -> Dict[str, tuple[str, str]]: def dependencies(self, extras_requested: Optional[Set[str]] = None) -> Set[str]: dependency_set = set() - for wheel_req in self.metadata.get_all('Requires-Dist', list()): + for wheel_req in self.metadata.get_all('Requires-Dist', []): req = pkg_resources.Requirement(wheel_req) # type: ignore if req.marker is None or any(