Skip to content

Commit 3ea1a55

Browse files
native-apinative-api
authored
Add prebuilt CPython download index (#2)
* Add prebuilt CPython download index * Validate prebuilt artifacts before updating * Reject invalid prebuilt archive names
2 parents 637dfb0 + 5e4e2c2 commit 3ea1a55

3 files changed

Lines changed: 159 additions & 2 deletions

File tree

‎pythons/index.html‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,14 @@ <h3>Welcome to the pyenv's official mirror of Python archives.</h3>
4141
$ pyenv install 2.7.7
4242
</code></pre>
4343

44-
<p>The filenames of the archives must be the md5sum of its content.
44+
<h3>Prebuilt CPython</h3>
45+
46+
<p>Prebuilt CPython archives and their python-build definitions use stable filenames.</p>
47+
48+
<ul id="prebuilt-cpython">
49+
</ul>
50+
51+
<p>The filenames of the source archives below are their SHA-256 checksums.
4552
The following links are the contents of the pyenv's official mirror of Python archives.</p>
4653

4754
<h3>Stackless 2.7</h3>

‎pythons/test/update.sh‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
cd "${BASH_SOURCE%/*}/.."
6+
7+
fail() {
8+
echo "$1" >&2
9+
exit 1
10+
}
11+
12+
write_meta() {
13+
cat > binaries/3.14.0-ubuntu-24.04-x86_64.meta <<EOF
14+
# pyenv-binary metadata
15+
version=3.14.0
16+
os=Linux
17+
arch=x86_64
18+
platform=linux-x86_64
19+
distro=ubuntu 24.04
20+
libc=glibc 2.39
21+
build_prefix=/tmp/pyenv/versions/3.14.0-ubuntu-24.04-x86_64
22+
archive=$1
23+
EOF
24+
}
25+
26+
tmpdir="$(mktemp -d)"
27+
trap 'rm -rf "$tmpdir"' EXIT
28+
29+
cp update.sh index.html "$tmpdir"
30+
mkdir "$tmpdir/binaries"
31+
printf archive-data > "$tmpdir/binaries/3.14.0-ubuntu-24.04-x86_64.tar.gz"
32+
printf definition > "$tmpdir/binaries/3.14.0-ubuntu-24.04-x86_64"
33+
34+
cd "$tmpdir"
35+
write_meta 3.14.0-ubuntu-24.04-x86_64.tar.gz
36+
bash ./update.sh >/dev/null 2>&1 || fail "update failed"
37+
38+
sha=8a6111c3ca752ed6d5f8e8a6daa3ba4b8c3b0bf55f00a87ac2b285024ef87e5f
39+
[ "binaries/3.14.0-ubuntu-24.04-x86_64.tar.gz" -ef "$sha" ] ||
40+
fail "checksum path is not a hardlink to the archive"
41+
42+
entry='<li><a href="binaries/3.14.0-ubuntu-24.04-x86_64.tar.gz">3.14.0-ubuntu-24.04-x86_64.tar.gz</a> (<a href="binaries/3.14.0-ubuntu-24.04-x86_64">definition</a>)</li>'
43+
grep -Fqx "$entry" index.html || fail "prebuilt archive is missing from index.html"
44+
45+
cp index.html index.before
46+
bash ./update.sh >/dev/null 2>&1 || fail "second update failed"
47+
cmp -s index.before index.html || fail "second update changed index.html"
48+
[ "$(grep -Fxc "$entry" index.html)" -eq 1 ] || fail "prebuilt archive is listed more than once"
49+
50+
bad_name=$'bad\nname'
51+
printf 'archive=bad\narchive=name.tar.gz\n' > "binaries/$bad_name.meta"
52+
printf archive-data > "binaries/$bad_name.tar.gz"
53+
printf definition > "binaries/$bad_name"
54+
rm "$sha"
55+
if output="$(bash ./update.sh 2>&1)"; then
56+
fail "update accepted a control character in an archive name"
57+
fi
58+
case "$output" in
59+
*"Invalid archive name in binaries/"*) ;;
60+
*) fail "update did not reject the invalid archive name during validation" ;;
61+
esac
62+
[ ! -e "$sha" ] || fail "update created a checksum link before name validation completed"
63+
rm "binaries/$bad_name.meta" "binaries/$bad_name.tar.gz" "binaries/$bad_name"
64+
65+
mkdir source
66+
printf source-data > source/example.tar.gz
67+
printf '<li><a href="">example.tar.gz</a></li>\n' >> index.html
68+
cp index.html index.before
69+
cp binaries/3.14.0-ubuntu-24.04-x86_64.meta binaries/z-invalid.meta
70+
sed -i 's/^archive=.*/archive=invalid.tar.gz/' binaries/z-invalid.meta
71+
if bash ./update.sh >/dev/null 2>&1; then
72+
fail "update accepted invalid metadata"
73+
fi
74+
[ ! -e "$sha" ] || fail "update created a binary checksum link before validation completed"
75+
source_sha=6bb69d845f4a714ca982e2903d2c03fabeb2448b67185bca413d3efddea9397c
76+
[ ! -e "$source_sha" ] || fail "update created a source checksum link before validation completed"
77+
cmp -s index.before index.html || fail "update changed index.html before validation completed"
78+
rm -rf source binaries/z-invalid.meta
79+
80+
write_meta ../outside.tar.gz
81+
printf outside > outside.tar.gz
82+
if bash ./update.sh >/dev/null 2>&1; then
83+
fail "update accepted an archive outside binaries"
84+
fi
85+
86+
write_meta 3.14.0-ubuntu-24.04-x86_64.tar.gz
87+
rm binaries/3.14.0-ubuntu-24.04-x86_64
88+
if bash ./update.sh >/dev/null 2>&1; then
89+
fail "update accepted a missing definition"
90+
fi
91+
92+
printf definition > binaries/3.14.0-ubuntu-24.04-x86_64
93+
rm binaries/3.14.0-ubuntu-24.04-x86_64.tar.gz
94+
if bash ./update.sh >/dev/null 2>&1; then
95+
fail "update accepted a missing archive"
96+
fi
97+
98+
echo "ok"

‎pythons/update.sh‎

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,12 @@
99
# 2. Download the archive from origin and save it in `./source`
1010
# 3. Run `./update.sh`
1111
# 4. Check diff of `./index.html` if the checksum is calculated properly
12-
# 5. Commit files with name of `md5sum` and `sha256sum` of the archive
12+
# 5. Commit the checksum-named archive files
1313
# 6. Push changes to the origin
1414
#
15+
# To add a prebuilt CPython, copy the archive, metadata and definition produced
16+
# by `pyenv binary package` to `./binaries`. The index entry is generated here.
17+
#
1518

1619
set -e
1720
set -x
@@ -47,7 +50,39 @@ compute_md5() {
4750
fi
4851
}
4952

53+
tmpdir="$(mktemp -d)"
54+
trap 'rm -rf "$tmpdir"' EXIT
55+
list="$tmpdir/list"
56+
links="$tmpdir/links"
57+
: > "$list"
58+
: > "$links"
59+
60+
for meta in binaries/*.meta; do
61+
[ -e "$meta" ] || continue
62+
name="$(basename "$meta" .meta)"
63+
archive="$(sed -n 's/^archive=//p' "$meta")"
64+
case "$archive" in
65+
"" | *[!A-Za-z0-9._-]*)
66+
echo "Invalid archive name in $meta" >&2
67+
exit 1
68+
;;
69+
esac
70+
if [ "$archive" != "$name.tar.gz" ]; then
71+
echo "Invalid archive in $meta" >&2
72+
exit 1
73+
fi
74+
if [ ! -f "binaries/$archive" ] || [ ! -f "binaries/$name" ]; then
75+
echo "Missing archive or definition for $name" >&2
76+
exit 1
77+
fi
78+
sha="$(compute_sha2 < "binaries/$archive")"
79+
printf '%s\n%s\n' "$archive" "$sha" >> "$links"
80+
printf '<li><a href="binaries/%s">%s</a> (<a href="binaries/%s">definition</a>)</li>\n' \
81+
"$archive" "$archive" "$name" >> "$list"
82+
done
83+
5084
for file in source/*; do
85+
[ -e "$file" ] || continue
5186
base="$(basename "$file")"
5287
#md5="$(compute_md5 < "$file")"
5388
sha="$(compute_sha2 < "$file")"
@@ -56,4 +91,21 @@ for file in source/*; do
5691
sed -i -e "/>$base</s/^.*$/<li><a href=\"$sha\">$base<\/a><\/li>/" index.html
5792
done
5893

94+
while IFS= read -r archive && IFS= read -r sha; do
95+
ln -f "binaries/$archive" "$sha"
96+
done < "$links"
97+
98+
awk -v list="$list" '
99+
/<ul id="prebuilt-cpython">/ {
100+
print
101+
while ((getline line < list) > 0) print line
102+
close(list)
103+
replacing = 1
104+
next
105+
}
106+
replacing && /<\/ul>/ { replacing = 0 }
107+
!replacing { print }
108+
' index.html > index.html.tmp
109+
mv index.html.tmp index.html
110+
59111
# vim:set ft=sh :

0 commit comments

Comments
 (0)