From 3a80e6a0ab7d6bd2a76796ec985500b3fe2e340a Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Tue, 11 Mar 2025 19:29:04 +0800 Subject: [PATCH 01/15] add test scenario for sdists across platforms --- .../parse_requirements_tests.bzl | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index c50482127b..740e5606b2 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -80,6 +80,12 @@ foo==0.0.3 --hash=sha256:deadbaaf "requirements_windows": """\ foo[extra]==0.0.2 --hash=sha256:deadbeef bar==0.0.1 --hash=sha256:deadb00f +""", + "requirements_sdist_different_hashes": """\ +foo==0.0.1 --hash=sha256:deadbeef --hash=sha256:cafebabe +""", + "requirements_sdist_different_hashes_linux": """\ +foo==0.0.1 --hash=sha256:deadbaaf --hash=sha256:cafebabe """, } @@ -623,6 +629,105 @@ def _test_optional_hash(env): _tests.append(_test_optional_hash) +def _test_sdist_different_hashes(env): + """Test that sdists with same hash but wheels with different hashes across platforms are handled correctly.""" + def _mock_get_index_urls(_, distributions): + return { + "foo": struct( + whls = { + "deadbeef": struct( + filename = "foo-0.0.1-py3-none-win_amd64.whl", + url = "https://pypi.org/foo-0.0.1-win.whl", + sha256 = "deadbeef", + yanked = False, + ), + "deadbaaf": struct( + filename = "foo-0.0.1-py3-none-manylinux_2_17_x86_64.whl", + url = "https://pypi.org/foo-0.0.1-linux.whl", + sha256 = "deadbaaf", + yanked = False, + ), + }, + sdists = { + "cafebabe": struct( + filename = "foo-0.0.1.tar.gz", + url = "https://pypi.org/foo-0.0.1.tar.gz", + sha256 = "cafebabe", + yanked = False, + ), + }, + ), + } + + got = parse_requirements( + ctx = _mock_ctx(), + requirements_by_platform = { + "requirements_sdist_different_hashes": ["windows_x86_64"], + "requirements_sdist_different_hashes_linux": ["linux_x86_64"], + }, + get_index_urls = _mock_get_index_urls, + ) + env.expect.that_dict(got).contains_exactly({ + "foo": [ + struct( + distribution = "foo", + extra_pip_args = [], + is_exposed = True, + sdist = struct( + filename = "foo-0.0.1.tar.gz", + url = "https://pypi.org/foo-0.0.1.tar.gz", + sha256 = "cafebabe", + yanked = False, + ), + srcs = struct( + marker = "", + requirement = "foo==0.0.1", + requirement_line = "foo==0.0.1 --hash=sha256:deadbaaf --hash=sha256:cafebabe", + shas = ["deadbaaf", "cafebabe"], + version = "0.0.1", + ), + target_platforms = ["linux_x86_64"], + whls = [ + struct( + filename = "foo-0.0.1-py3-none-manylinux_2_17_x86_64.whl", + url = "https://pypi.org/foo-0.0.1-linux.whl", + sha256 = "deadbaaf", + yanked = False, + ), + ], + ), + struct( + distribution = "foo", + extra_pip_args = [], + is_exposed = True, + sdist = struct( + filename = "foo-0.0.1.tar.gz", + url = "https://pypi.org/foo-0.0.1.tar.gz", + sha256 = "cafebabe", + yanked = False, + ), + srcs = struct( + marker = "", + requirement = "foo==0.0.1", + requirement_line = "foo==0.0.1 --hash=sha256:deadbeef --hash=sha256:cafebabe", + shas = ["deadbeef", "cafebabe"], + version = "0.0.1", + ), + target_platforms = ["windows_x86_64"], + whls = [ + struct( + filename = "foo-0.0.1-py3-none-win_amd64.whl", + url = "https://pypi.org/foo-0.0.1-win.whl", + sha256 = "deadbeef", + yanked = False, + ), + ], + ), + ], + }) + +_tests.append(_test_sdist_different_hashes) + def parse_requirements_test_suite(name): """Create the test suite. From 5adac71754a637aaf550168642ad2bf3dedc1cf1 Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Tue, 11 Mar 2025 20:21:48 +0800 Subject: [PATCH 02/15] fix tests --- .../parse_requirements/parse_requirements_tests.bzl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 740e5606b2..4bdee2e34e 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -662,8 +662,8 @@ def _test_sdist_different_hashes(env): got = parse_requirements( ctx = _mock_ctx(), requirements_by_platform = { - "requirements_sdist_different_hashes": ["windows_x86_64"], - "requirements_sdist_different_hashes_linux": ["linux_x86_64"], + "requirements_sdist_different_hashes": ["cp315_windows_x86_64"], + "requirements_sdist_different_hashes_linux": ["cp315_linux_x86_64"], }, get_index_urls = _mock_get_index_urls, ) @@ -683,10 +683,10 @@ def _test_sdist_different_hashes(env): marker = "", requirement = "foo==0.0.1", requirement_line = "foo==0.0.1 --hash=sha256:deadbaaf --hash=sha256:cafebabe", - shas = ["deadbaaf", "cafebabe"], + shas = ["cafebabe", "deadbaaf"], version = "0.0.1", ), - target_platforms = ["linux_x86_64"], + target_platforms = ["cp315_linux_x86_64"], whls = [ struct( filename = "foo-0.0.1-py3-none-manylinux_2_17_x86_64.whl", @@ -710,10 +710,10 @@ def _test_sdist_different_hashes(env): marker = "", requirement = "foo==0.0.1", requirement_line = "foo==0.0.1 --hash=sha256:deadbeef --hash=sha256:cafebabe", - shas = ["deadbeef", "cafebabe"], + shas = ["cafebabe", "deadbeef"], version = "0.0.1", ), - target_platforms = ["windows_x86_64"], + target_platforms = ["cp315_windows_x86_64"], whls = [ struct( filename = "foo-0.0.1-py3-none-win_amd64.whl", From 7a30c1f7ba8b159aa00a9d8e7f71efffbabf5837 Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Tue, 11 Mar 2025 20:46:17 +0800 Subject: [PATCH 03/15] sketch for new merge logic --- python/private/pypi/parse_requirements.bzl | 55 ++++++++++++++++--- .../parse_requirements_tests.bzl | 22 ++++++-- 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 3280ce8df1..18eae6ecb4 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -207,26 +207,65 @@ def parse_requirements( # Return normalized names ret_requirements = ret.setdefault(normalize_name(whl_name), []) + # First, find if there's a common sdist across all platforms + common_sdist = None + common_sdist_hash = None + all_platforms = [] for r in sorted(reqs.values(), key = lambda r: r.requirement_line): + target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) + all_platforms.extend(target_platforms) whls, sdist = _add_dists( requirement = r, index_urls = index_urls.get(whl_name), logger = logger, ) - - target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) + if sdist: + if not common_sdist: + common_sdist = sdist + common_sdist_hash = sdist.sha256 + elif sdist.sha256 == common_sdist_hash: + continue + else: + common_sdist = None + break + + # If we found a common sdist, add it as a separate entry + if common_sdist: + # Take the first requirement's info as base since version should be same + base_req = sorted(reqs.values(), key = lambda r: r.requirement_line)[0] ret_requirements.append( struct( - distribution = r.distribution, - srcs = r.srcs, - target_platforms = sorted(target_platforms), - extra_pip_args = r.extra_pip_args, - whls = whls, - sdist = sdist, + distribution = base_req.distribution, + srcs = base_req.srcs, + target_platforms = sorted(all_platforms), + extra_pip_args = base_req.extra_pip_args, + whls = [], + sdist = common_sdist, is_exposed = is_exposed, ), ) + # Now add platform-specific entries with wheels + for r in sorted(reqs.values(), key = lambda r: r.requirement_line): + whls, sdist = _add_dists( + requirement = r, + index_urls = index_urls.get(whl_name), + logger = logger, + ) + if not common_sdist or len(whls) > 0: # Only add entries that have wheels + target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) + ret_requirements.append( + struct( + distribution = r.distribution, + srcs = r.srcs, + target_platforms = sorted(target_platforms), + extra_pip_args = r.extra_pip_args, + whls = whls, + sdist = None if common_sdist else sdist, # No sdist in wheel-specific entries + is_exposed = is_exposed, + ), + ) + if logger: logger.debug(lambda: "Will configure whl repos: {}".format(ret.keys())) diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 4bdee2e34e..d63a49d96f 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -686,6 +686,21 @@ def _test_sdist_different_hashes(env): shas = ["cafebabe", "deadbaaf"], version = "0.0.1", ), + target_platforms = ["cp315_linux_x86_64", "cp315_windows_x86_64"], + whls = [], + ), + struct( + distribution = "foo", + extra_pip_args = [], + is_exposed = True, + sdist = None, + srcs = struct( + marker = "", + requirement = "foo==0.0.1", + requirement_line = "foo==0.0.1 --hash=sha256:deadbaaf --hash=sha256:cafebabe", + shas = ["cafebabe", "deadbaaf"], + version = "0.0.1", + ), target_platforms = ["cp315_linux_x86_64"], whls = [ struct( @@ -700,12 +715,7 @@ def _test_sdist_different_hashes(env): distribution = "foo", extra_pip_args = [], is_exposed = True, - sdist = struct( - filename = "foo-0.0.1.tar.gz", - url = "https://pypi.org/foo-0.0.1.tar.gz", - sha256 = "cafebabe", - yanked = False, - ), + sdist = None, srcs = struct( marker = "", requirement = "foo==0.0.1", From dc827aabe988d110ebc46cfd394516361962631e Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Tue, 11 Mar 2025 21:17:32 +0800 Subject: [PATCH 04/15] simplify logic assuming one sdist always per version --- python/private/pypi/parse_requirements.bzl | 66 +++++++------------ .../parse_requirements_tests.bzl | 40 +++++------ 2 files changed, 44 insertions(+), 62 deletions(-) diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 18eae6ecb4..8efc441d39 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -207,65 +207,47 @@ def parse_requirements( # Return normalized names ret_requirements = ret.setdefault(normalize_name(whl_name), []) - # First, find if there's a common sdist across all platforms - common_sdist = None - common_sdist_hash = None all_platforms = [] + common_sdist = None + for r in sorted(reqs.values(), key = lambda r: r.requirement_line): - target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) - all_platforms.extend(target_platforms) whls, sdist = _add_dists( requirement = r, index_urls = index_urls.get(whl_name), logger = logger, ) - if sdist: - if not common_sdist: - common_sdist = sdist - common_sdist_hash = sdist.sha256 - elif sdist.sha256 == common_sdist_hash: - continue - else: - common_sdist = None - break - - # If we found a common sdist, add it as a separate entry + + if sdist and not common_sdist: + common_sdist = sdist + + target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) + all_platforms.extend(target_platforms) + + ret_requirements.append( + struct( + distribution = r.distribution, + srcs = r.srcs, + target_platforms = sorted(target_platforms), + extra_pip_args = r.extra_pip_args, + whls = whls, + sdist = None, + is_exposed = is_exposed, + ), + ) + if common_sdist: - # Take the first requirement's info as base since version should be same - base_req = sorted(reqs.values(), key = lambda r: r.requirement_line)[0] ret_requirements.append( struct( - distribution = base_req.distribution, - srcs = base_req.srcs, + distribution = r.distribution, + srcs = r.srcs, target_platforms = sorted(all_platforms), - extra_pip_args = base_req.extra_pip_args, + extra_pip_args = r.extra_pip_args, whls = [], sdist = common_sdist, is_exposed = is_exposed, ), ) - # Now add platform-specific entries with wheels - for r in sorted(reqs.values(), key = lambda r: r.requirement_line): - whls, sdist = _add_dists( - requirement = r, - index_urls = index_urls.get(whl_name), - logger = logger, - ) - if not common_sdist or len(whls) > 0: # Only add entries that have wheels - target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) - ret_requirements.append( - struct( - distribution = r.distribution, - srcs = r.srcs, - target_platforms = sorted(target_platforms), - extra_pip_args = r.extra_pip_args, - whls = whls, - sdist = None if common_sdist else sdist, # No sdist in wheel-specific entries - is_exposed = is_exposed, - ), - ) - if logger: logger.debug(lambda: "Will configure whl repos: {}".format(ret.keys())) diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index d63a49d96f..9764671eba 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -669,26 +669,6 @@ def _test_sdist_different_hashes(env): ) env.expect.that_dict(got).contains_exactly({ "foo": [ - struct( - distribution = "foo", - extra_pip_args = [], - is_exposed = True, - sdist = struct( - filename = "foo-0.0.1.tar.gz", - url = "https://pypi.org/foo-0.0.1.tar.gz", - sha256 = "cafebabe", - yanked = False, - ), - srcs = struct( - marker = "", - requirement = "foo==0.0.1", - requirement_line = "foo==0.0.1 --hash=sha256:deadbaaf --hash=sha256:cafebabe", - shas = ["cafebabe", "deadbaaf"], - version = "0.0.1", - ), - target_platforms = ["cp315_linux_x86_64", "cp315_windows_x86_64"], - whls = [], - ), struct( distribution = "foo", extra_pip_args = [], @@ -733,6 +713,26 @@ def _test_sdist_different_hashes(env): ), ], ), + struct( + distribution = "foo", + extra_pip_args = [], + is_exposed = True, + sdist = struct( + filename = "foo-0.0.1.tar.gz", + url = "https://pypi.org/foo-0.0.1.tar.gz", + sha256 = "cafebabe", + yanked = False, + ), + srcs = struct( + marker = "", + requirement = "foo==0.0.1", + requirement_line = "foo==0.0.1 --hash=sha256:deadbeef --hash=sha256:cafebabe", + shas = ["cafebabe", "deadbeef"], + version = "0.0.1", + ), + target_platforms = ["cp315_linux_x86_64", "cp315_windows_x86_64"], + whls = [], + ), ], }) From 22cce7922957c6c6a1e5ce366a2b7131879fcf00 Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Tue, 11 Mar 2025 21:24:48 +0800 Subject: [PATCH 05/15] unsure: address test failure --- tests/pypi/extension/extension_tests.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 3a91c7b108..0240234526 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -703,6 +703,7 @@ pip_fallback==0.0.1 filename = "simple-0.0.1-py3-none-any.whl", target_platforms = None, version = "3.15", + target_platforms = ("cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"), ), ], "pypi_315_simple_sdist_deadbeef": [ @@ -721,6 +722,7 @@ pip_fallback==0.0.1 filename = "some-other-pkg-0.0.1-py3-none-any.whl", target_platforms = None, version = "3.15", + target_platforms = ("cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"), ), ], }, From ddfecc44affb4824d16756b4156a9d30c6458914 Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Fri, 4 Apr 2025 11:11:39 +0800 Subject: [PATCH 06/15] fix merge --- tests/pypi/extension/extension_tests.bzl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 0240234526..d275bbf780 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -701,9 +701,8 @@ pip_fallback==0.0.1 struct( config_setting = None, filename = "simple-0.0.1-py3-none-any.whl", - target_platforms = None, - version = "3.15", target_platforms = ("cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"), + version = "3.15", ), ], "pypi_315_simple_sdist_deadbeef": [ @@ -720,9 +719,8 @@ pip_fallback==0.0.1 struct( config_setting = None, filename = "some-other-pkg-0.0.1-py3-none-any.whl", - target_platforms = None, - version = "3.15", target_platforms = ("cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"), + version = "3.15", ), ], }, From 64f355ca3a4dab7e9c1269f08cfb7373c279bd5b Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Fri, 4 Apr 2025 11:28:37 +0800 Subject: [PATCH 07/15] use sha dictionary instead --- python/private/pypi/parse_requirements.bzl | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 8efc441d39..e6223c1f01 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -206,9 +206,7 @@ def parse_requirements( # Return normalized names ret_requirements = ret.setdefault(normalize_name(whl_name), []) - - all_platforms = [] - common_sdist = None + sdists_by_sha = {} for r in sorted(reqs.values(), key = lambda r: r.requirement_line): whls, sdist = _add_dists( @@ -217,11 +215,16 @@ def parse_requirements( logger = logger, ) - if sdist and not common_sdist: - common_sdist = sdist + if sdist: + sha = sdist.sha256 + if sha not in sdists_by_sha: + sdists_by_sha[sha] = struct( + sdist = sdist, + platforms = [], + ) + sdists_by_sha[sha].platforms.extend(r.target_platforms) target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) - all_platforms.extend(target_platforms) ret_requirements.append( struct( @@ -235,15 +238,15 @@ def parse_requirements( ), ) - if common_sdist: + for sha, sdist_info in sdists_by_sha.items(): ret_requirements.append( struct( distribution = r.distribution, srcs = r.srcs, - target_platforms = sorted(all_platforms), + target_platforms = sorted(sdist_info.platforms), extra_pip_args = r.extra_pip_args, whls = [], - sdist = common_sdist, + sdist = sdist_info.sdist, is_exposed = is_exposed, ), ) From 16e0bc6f35405448e36b0e842d6a101eeccf92a2 Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Fri, 4 Apr 2025 11:57:12 +0800 Subject: [PATCH 08/15] fix test --- tests/pypi/extension/extension_tests.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index d275bbf780..1fe246416e 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -709,7 +709,7 @@ pip_fallback==0.0.1 struct( config_setting = None, filename = "simple-0.0.1.tar.gz", - target_platforms = None, + target_platforms = ("cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"), version = "3.15", ), ], @@ -719,7 +719,7 @@ pip_fallback==0.0.1 struct( config_setting = None, filename = "some-other-pkg-0.0.1-py3-none-any.whl", - target_platforms = ("cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"), + target_platforms = None, version = "3.15", ), ], From fa04ed599a129ae161755f40eb8c21a59554406b Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Fri, 4 Apr 2025 13:55:51 +0800 Subject: [PATCH 09/15] attempt to make tests pass --- python/private/pypi/parse_requirements.bzl | 18 ++++++++++++------ .../parse_requirements_tests.bzl | 7 +++++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index e6223c1f01..e7d2e0fc20 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -215,16 +215,22 @@ def parse_requirements( logger = logger, ) + target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) + if sdist: sha = sdist.sha256 if sha not in sdists_by_sha: sdists_by_sha[sha] = struct( + distribution = r.distribution, + srcs = r.srcs, + extra_pip_args = r.extra_pip_args, sdist = sdist, platforms = [], ) - sdists_by_sha[sha].platforms.extend(r.target_platforms) - - target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) + sdists_by_sha[sha].platforms.extend(target_platforms) + + if len(whls) == 0: + continue ret_requirements.append( struct( @@ -241,10 +247,10 @@ def parse_requirements( for sha, sdist_info in sdists_by_sha.items(): ret_requirements.append( struct( - distribution = r.distribution, - srcs = r.srcs, + distribution = sdist_info.distribution, + srcs = sdist_info.srcs, target_platforms = sorted(sdist_info.platforms), - extra_pip_args = r.extra_pip_args, + extra_pip_args = sdist_info.extra_pip_args, whls = [], sdist = sdist_info.sdist, is_exposed = is_exposed, diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 9764671eba..4eedaa226e 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -679,6 +679,7 @@ def _test_sdist_different_hashes(env): requirement = "foo==0.0.1", requirement_line = "foo==0.0.1 --hash=sha256:deadbaaf --hash=sha256:cafebabe", shas = ["cafebabe", "deadbaaf"], + url = "", version = "0.0.1", ), target_platforms = ["cp315_linux_x86_64"], @@ -701,6 +702,7 @@ def _test_sdist_different_hashes(env): requirement = "foo==0.0.1", requirement_line = "foo==0.0.1 --hash=sha256:deadbeef --hash=sha256:cafebabe", shas = ["cafebabe", "deadbeef"], + url = "", version = "0.0.1", ), target_platforms = ["cp315_windows_x86_64"], @@ -726,8 +728,9 @@ def _test_sdist_different_hashes(env): srcs = struct( marker = "", requirement = "foo==0.0.1", - requirement_line = "foo==0.0.1 --hash=sha256:deadbeef --hash=sha256:cafebabe", - shas = ["cafebabe", "deadbeef"], + requirement_line = "foo==0.0.1 --hash=sha256:deadbaaf --hash=sha256:cafebabe", + shas = ["cafebabe", "deadbaaf"], + url = "", version = "0.0.1", ), target_platforms = ["cp315_linux_x86_64", "cp315_windows_x86_64"], From c56d667ad9ef5e6e2d9532bcb1219456faca4d87 Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Fri, 4 Apr 2025 13:58:46 +0800 Subject: [PATCH 10/15] remove extraneous whitespace --- python/private/pypi/parse_requirements.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index e7d2e0fc20..568559a577 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -214,7 +214,7 @@ def parse_requirements( index_urls = index_urls.get(whl_name), logger = logger, ) - + target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) if sdist: From 4fc4b2ac17007adbc1ec407c4ab343452048e29d Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Fri, 4 Apr 2025 14:09:06 +0800 Subject: [PATCH 11/15] remove platforms --- python/private/pypi/parse_requirements.bzl | 1 - tests/pypi/extension/extension_tests.bzl | 4 ++-- tests/pypi/parse_requirements/parse_requirements_tests.bzl | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 568559a577..bd0b9ec6b5 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -227,7 +227,6 @@ def parse_requirements( sdist = sdist, platforms = [], ) - sdists_by_sha[sha].platforms.extend(target_platforms) if len(whls) == 0: continue diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 1fe246416e..1f7acdf9d0 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -709,7 +709,7 @@ pip_fallback==0.0.1 struct( config_setting = None, filename = "simple-0.0.1.tar.gz", - target_platforms = ("cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"), + target_platforms = None, version = "3.15", ), ], @@ -766,7 +766,7 @@ pip_fallback==0.0.1 }, "pypi_315_simple_sdist_deadbeef": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"], + "experimental_target_platforms": [], "extra_pip_args": ["--extra-args-for-sdist-building"], "filename": "simple-0.0.1.tar.gz", "python_interpreter_target": "unit_test_interpreter_target", diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 4eedaa226e..7471b8c1d0 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -234,7 +234,7 @@ def _test_direct_urls(env): version = "", url = "https://example.org/qux-1.0.tar.gz", ), - target_platforms = ["linux_x86_64"], + target_platforms = [], whls = [], ), ], @@ -733,7 +733,7 @@ def _test_sdist_different_hashes(env): url = "", version = "0.0.1", ), - target_platforms = ["cp315_linux_x86_64", "cp315_windows_x86_64"], + target_platforms = [], whls = [], ), ], From d24c7aaa2e8496cb2bb7cec2b61718c7b66e60c7 Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Fri, 4 Apr 2025 14:13:17 +0800 Subject: [PATCH 12/15] Revert "remove platforms" This reverts commit 4fc4b2ac17007adbc1ec407c4ab343452048e29d. --- python/private/pypi/parse_requirements.bzl | 1 + tests/pypi/extension/extension_tests.bzl | 4 ++-- tests/pypi/parse_requirements/parse_requirements_tests.bzl | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index bd0b9ec6b5..568559a577 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -227,6 +227,7 @@ def parse_requirements( sdist = sdist, platforms = [], ) + sdists_by_sha[sha].platforms.extend(target_platforms) if len(whls) == 0: continue diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 1f7acdf9d0..1fe246416e 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -709,7 +709,7 @@ pip_fallback==0.0.1 struct( config_setting = None, filename = "simple-0.0.1.tar.gz", - target_platforms = None, + target_platforms = ("cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"), version = "3.15", ), ], @@ -766,7 +766,7 @@ pip_fallback==0.0.1 }, "pypi_315_simple_sdist_deadbeef": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [], + "experimental_target_platforms": ["cp315_linux_aarch64", "cp315_linux_arm", "cp315_linux_ppc", "cp315_linux_s390x", "cp315_linux_x86_64", "cp315_osx_aarch64", "cp315_osx_x86_64", "cp315_windows_x86_64"], "extra_pip_args": ["--extra-args-for-sdist-building"], "filename": "simple-0.0.1.tar.gz", "python_interpreter_target": "unit_test_interpreter_target", diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 7471b8c1d0..4eedaa226e 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -234,7 +234,7 @@ def _test_direct_urls(env): version = "", url = "https://example.org/qux-1.0.tar.gz", ), - target_platforms = [], + target_platforms = ["linux_x86_64"], whls = [], ), ], @@ -733,7 +733,7 @@ def _test_sdist_different_hashes(env): url = "", version = "0.0.1", ), - target_platforms = [], + target_platforms = ["cp315_linux_x86_64", "cp315_windows_x86_64"], whls = [], ), ], From 6e98362be868102aa0bbeefc0e452406a25f89cb Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Fri, 4 Apr 2025 14:14:24 +0800 Subject: [PATCH 13/15] lint --- python/private/pypi/parse_requirements.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 568559a577..362ac68650 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -228,7 +228,7 @@ def parse_requirements( platforms = [], ) sdists_by_sha[sha].platforms.extend(target_platforms) - + if len(whls) == 0: continue @@ -243,7 +243,7 @@ def parse_requirements( is_exposed = is_exposed, ), ) - + for sha, sdist_info in sdists_by_sha.items(): ret_requirements.append( struct( From ee995ad560a5e1cb14506e12170c4b0f89e61a76 Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Fri, 4 Apr 2025 14:19:08 +0800 Subject: [PATCH 14/15] clean up unused variables --- python/private/pypi/parse_requirements.bzl | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 362ac68650..c979f68d41 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -219,15 +219,14 @@ def parse_requirements( if sdist: sha = sdist.sha256 - if sha not in sdists_by_sha: - sdists_by_sha[sha] = struct( - distribution = r.distribution, - srcs = r.srcs, - extra_pip_args = r.extra_pip_args, - sdist = sdist, - platforms = [], - ) - sdists_by_sha[sha].platforms.extend(target_platforms) + sdist_info = sdists_by_sha.setdefault(sha, struct( + distribution = r.distribution, + srcs = r.srcs, + extra_pip_args = r.extra_pip_args, + sdist = sdist, + platforms = [], + )) + sdist_info.platforms.extend(target_platforms) if len(whls) == 0: continue @@ -244,7 +243,7 @@ def parse_requirements( ), ) - for sha, sdist_info in sdists_by_sha.items(): + for sdist_info in sdists_by_sha.values(): ret_requirements.append( struct( distribution = sdist_info.distribution, From 6ed41736fd3ddce5eb8546a2cd2d02ae7190b16e Mon Sep 17 00:00:00 2001 From: Chris Chua Date: Fri, 4 Apr 2025 14:20:45 +0800 Subject: [PATCH 15/15] lint --- tests/pypi/parse_requirements/parse_requirements_tests.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 4eedaa226e..1dcc3265ec 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -631,6 +631,7 @@ _tests.append(_test_optional_hash) def _test_sdist_different_hashes(env): """Test that sdists with same hash but wheels with different hashes across platforms are handled correctly.""" + def _mock_get_index_urls(_, distributions): return { "foo": struct(