Skip to content

Commit d4cd71d

Browse files
tdyastdyas
authored
generate a multi-level PyPi index as per PEP 503 (#1)
[PEP 503](https://peps.python.org/pep-0503/) describes a multi-level PyPi index with a root index file with just links to the supported "projects" and project-specific index files with the actual links to release files. Update `generate_index.py` to generate a multi-level index and modify the test accordingly. The legacy links remains for now until we consider a migration path for how to remove them going forward.
1 parent 46f3188 commit d4cd71d

File tree

4 files changed

+145
-29
lines changed

4 files changed

+145
-29
lines changed

‎.github/workflows/pages.yaml

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,18 @@ jobs:
2525
runs-on: ubuntu-latest
2626
steps:
2727
- name: Checkout
28-
uses: actions/checkout@v3
28+
uses: actions/checkout@v4
2929
- name: Setup Pages
3030
uses: actions/configure-pages@v3
31+
- uses: actions/setup-python@v5
32+
with:
33+
python-version: '3.9'
3134
- name: Generate index
3235
run: |
33-
mkdir -p github-pages/simple
34-
pip install pygithub
35-
python generate_index.py > github-pages/simple/index.html
36+
python -m venv venv
37+
venv/bin/pip install -r ./requirements.txt
38+
mkdir -p github-pages
39+
venv/bin/python generate_index.py --url-path-prefix=/simple github-pages/simple
3640
env:
3741
GH_TOKEN: ${{ github.token }}
3842
- name: Upload artifact

‎generate_index.py

Lines changed: 127 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,136 @@
1+
from __future__ import annotations
2+
3+
import argparse
4+
from collections import defaultdict
15
import os
6+
from pathlib import Path
7+
import sys
8+
from typing import Any, Iterable
9+
import github.GitReleaseAsset
10+
from packaging.utils import parse_wheel_filename
11+
from packaging.version import Version
12+
from urllib.parse import urlparse
13+
from textwrap import dedent
214

315
import github
416

17+
##
18+
## Output a PEP 503 compliant package repository for Pants wheels.
19+
## See https://peps.python.org/pep-0503/
20+
##
21+
522

6-
def main() -> str:
7-
gh = github.Github(auth=github.Auth.Token(os.environ["GH_TOKEN"]))
23+
def get_pants_python_packages(gh: github.Github) -> dict[str, dict[Version, list[Any]]]:
824
repo = gh.get_repo("pantsbuild/pants")
9-
releases = repo.get_releases()
10-
index = "\n".join(
11-
[
12-
"<html>",
13-
"<body>",
14-
"<h1>Links for Pantsbuild Wheels</h1>",
15-
*(
16-
f'<a href="{asset.browser_download_url}">{asset.name}</a><br>'
17-
for release in releases
18-
if release.tag_name.startswith("release_2")
19-
for asset in release.assets
20-
if asset.name.endswith(".whl")
21-
),
22-
"</body>",
23-
"</html>",
24-
]
25-
)
26-
return index
25+
all_releases = repo.get_releases()
26+
27+
pants_wheel_assets = [
28+
asset
29+
for release in all_releases
30+
if release.tag_name.startswith("release_2")
31+
for asset in release.assets
32+
if asset.name.endswith(".whl")
33+
]
34+
35+
packages = defaultdict(lambda: defaultdict(list))
36+
37+
for asset in pants_wheel_assets:
38+
name, version, _build_tag, _tags = parse_wheel_filename(asset.name)
39+
packages[name][version].append(asset)
40+
41+
return packages
42+
43+
44+
def _legacy_flat_links(packages: dict[str, dict[Version, list[Any]]]) -> tuple[str, ...]:
45+
return [
46+
f'<a href="{asset.browser_download_url}">{asset.name}</a><br>'
47+
for package_versions in packages.values()
48+
for _, version_release_assets in sorted(package_versions.items(), reverse=True)
49+
for asset in version_release_assets
50+
]
51+
52+
# http://repository.example.com/simple/PACKAGE_NAME/
53+
def _write_package_specific_index(output_dir: Path, package_name: str, package_versions: dict[Version, list[Any]]) -> None:
54+
package_output_dir = output_dir / package_name
55+
package_output_dir.mkdir()
56+
57+
package_version_keys = sorted(package_versions.keys(), reverse=True)
58+
59+
with open(package_output_dir / "index.html", "w") as f:
60+
f.write(dedent(
61+
f"""\
62+
<!DOCTYPE html>
63+
<html>
64+
<body>
65+
<h1>Links for Pantsbuild Wheels - {package_name}</h1>
66+
<ul>
67+
"""
68+
))
69+
70+
for package_version_key in package_version_keys:
71+
package_version_assets = package_versions[package_version_key]
72+
package_version_assets.sort(key=lambda x: x.name)
73+
for asset in package_version_assets:
74+
f.write(f"""<li><a href="{asset.browser_download_url}">{asset.name}</a></li>\n""")
75+
76+
f.write(dedent(
77+
"""\
78+
</ul>
79+
</body>
80+
</html>
81+
"""
82+
))
83+
84+
85+
86+
def main(args):
87+
parser = argparse.ArgumentParser()
88+
parser.add_argument("--url-path-prefix", default="/", action="store")
89+
parser.add_argument("output_dir", action="store")
90+
opts = parser.parse_args(args)
91+
92+
github_client = github.Github(auth=github.Auth.Token(os.environ["GH_TOKEN"]))
93+
packages = get_pants_python_packages(github_client)
94+
package_names = sorted(packages.keys())
95+
96+
prefix = opts.url_path_prefix
97+
if prefix and prefix.endswith("/"):
98+
prefix = prefix[0:-1]
99+
100+
output_dir = Path(opts.output_dir)
101+
if output_dir.exists():
102+
raise Exception(f"Output directory `{output_dir}` already exists.")
103+
output_dir.mkdir(parents=True)
104+
105+
# http://repository.example.com/simple/
106+
with open(output_dir / "index.html", "w") as f:
107+
f.write(dedent(
108+
"""\
109+
<!DOCTYPE html>
110+
<html>
111+
<body>
112+
<h1>Links for Pantsbuild Wheels</h1>
113+
<ul>
114+
"""
115+
))
116+
117+
for package_name in package_names:
118+
f.write(f"""<li><a href="{prefix}/{package_name}/">{package_name}</a></li>\n""")
119+
120+
f.write("\n".join(_legacy_flat_links(packages)))
121+
122+
f.write(dedent(
123+
"""\
124+
</ul>
125+
</body>
126+
</html>
127+
"""
128+
))
129+
130+
# http://repository.example.com/simple/PACKAGE_NAME/
131+
for package_name in package_names:
132+
_write_package_specific_index(output_dir, package_name, packages[package_name])
27133

28134

29135
if __name__ == "__main__":
30-
print(main())
136+
sys.exit(main(sys.argv[1:]))

‎requirements.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1+
packaging==24.2
12
pygithub==2.5.0

‎test_generated_index.sh

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
#!/bin/bash
22

33
output_dir="$1"
4+
if [ -z "${output_dir}" ]; then
5+
echo "usage: $0 OUTPUT_DIRECTORY" 1>&2
6+
exit 1
7+
fi
48
if [ -e "${output_dir}" ]; then
59
echo "ERROR: Output directory exists. This script requires that it not exist already." 1>&2
610
exit 1
@@ -15,8 +19,9 @@ python3.9 -m venv "${venv_dir}"
1519
"${venv_dir}/bin/pip" install -r ./requirements.txt
1620

1721
# Generate the Pants PyPi-compatible index.
18-
mkdir -p "${output_dir}/public/simple"
19-
"${venv_dir}/bin/python" ./generate_index.py > "${output_dir}/public/simple/index.html"
22+
"${venv_dir}/bin/python" ./generate_index.py \
23+
--url-path-prefix=/simple \
24+
"${output_dir}/public/simple"
2025

2126
# Serve a copy of the generated index on port 8080.
2227
python3.9 -m http.server -d "${output_dir}/public" --bind 127.0.0.1 8080 &
@@ -27,8 +32,8 @@ python3.9 -m http.server -d "${output_dir}/public" --bind 127.0.0.1 8080 &
2732
pants_venv_dir="${output_dir}/pants-venv"
2833
python3.9 -m venv "${pants_venv_dir}"
2934
"${pants_venv_dir}/bin/pip" install -vv \
30-
--extra-index-url=http://127.0.0.1:8080/simple/ \
31-
pantsbuild.pants==2.18.0a0
35+
--extra-index-url=http://127.0.0.1:8080/simple/ \
36+
pantsbuild.pants==2.23.0
3237

3338
# Verify that the Pants console script is in the expected location.
3439
if [ ! -f "${pants_venv_dir}/bin/pants" ]; then

0 commit comments

Comments
 (0)