Skip to content

Commit 790ac2e

Browse files
darylcdarylc
committed
ci(rpi-imager): parallelize per-platform image download/hash into a matrix job
1 parent a8f128d commit 790ac2e

1 file changed

Lines changed: 140 additions & 111 deletions

File tree

‎.github/workflows/update-rpi-imager-json.yml‎

Lines changed: 140 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@
1212
# existing entry (e.g. Pi64, which isn't published to rpi-imager today) is
1313
# skipped with a warning rather than guessing at a new entry's "devices"
1414
# list, icon, etc.
15+
#
16+
# The four platforms' downloads/hashing run as parallel matrix jobs (each
17+
# multi-GB image is fetched+extracted+hashed independently) rather than one
18+
# long sequential loop -- they don't depend on each other. A final job
19+
# gathers the per-platform results and does the JSON update + PR.
1520
#############################################################################
1621

1722
name: Update RPi Imager JSON
@@ -38,16 +43,13 @@ permissions:
3843
pull-requests: write
3944

4045
jobs:
41-
update:
46+
resolve:
4247
runs-on: ubuntu-latest
4348
env:
4449
GH_TOKEN: ${{ github.token }}
50+
outputs:
51+
tag: ${{ steps.release.outputs.tag }}
4552
steps:
46-
- name: Checkout master
47-
uses: actions/checkout@v5
48-
with:
49-
ref: master
50-
5153
- name: Resolve release
5254
id: release
5355
run: |
@@ -81,108 +83,130 @@ jobs:
8183
# tag_name/published_at -- using those here previously overwrote
8284
# correct "9.5.3" entries back down to "9.5" (PR #2772).
8385
84-
- name: Download platform images and compute hashes
86+
- name: Upload release.json
87+
uses: actions/upload-artifact@v4
88+
with:
89+
name: release-json
90+
path: release.json
91+
retention-days: 1
92+
93+
hashes:
94+
needs: resolve
95+
runs-on: ubuntu-latest
96+
strategy:
97+
fail-fast: false
98+
matrix:
99+
# Suffix must match the trailing "-<suffix>.img.zip" of the asset
100+
# filename exactly (a plain substring match would let "Pi" match
101+
# "Pi64" too).
102+
suffix: [Pi, Pi64, BB64, BBB]
103+
steps:
104+
- name: Download release.json
105+
uses: actions/download-artifact@v4
106+
with:
107+
name: release-json
108+
109+
- name: Download image and compute hashes
85110
id: hashes
111+
env:
112+
SUFFIX: ${{ matrix.suffix }}
86113
run: |
87114
set -euo pipefail
88115
mkdir -p work
89-
# Suffix must match the trailing "-<suffix>.img.zip" of the asset
90-
# filename exactly (a plain substring match would let "Pi" match
91-
# "Pi64" too).
92-
for SUFFIX in Pi Pi64 BB64 BBB; do
93-
MATCH=$(jq -c --arg s "$SUFFIX" \
94-
'[.assets[] | select(.name | test("-" + $s + "\\.img\\.zip$"))] | sort_by(.updated_at) | last // empty' \
95-
release.json)
96-
97-
if [ -z "$MATCH" ]; then
98-
echo "No .img.zip asset for platform $SUFFIX in this release -- skipping."
99-
continue
100-
fi
101-
102-
NAME=$(jq -r '.name' <<<"$MATCH")
103-
URL=$(jq -r '.browser_download_url' <<<"$MATCH")
104-
# Per-asset, not per-release: this repo re-uploads patch bumps
105-
# (9.5, 9.5.1, ..., 9.5.3) onto the SAME release/tag over time,
106-
# so the release object's own tag_name/published_at are stale
107-
# for anything past the first upload (see the note above).
108-
ASSET_DATE=$(jq -r '(.updated_at // .created_at)' <<<"$MATCH" | cut -d'T' -f1)
109-
# Asset filenames are FPP-v<VERSION>-<SUFFIX>.img.zip (see
110-
# SD/build-image-pi.sh's OUT_IMG); strip the fixed prefix/suffix
111-
# to recover VERSION even though it can differ from the release
112-
# tag_name (FPP-v9.5.3-Pi.img.zip on tag "9.5").
113-
ASSET_VERSION="${NAME#FPP-v}"
114-
ASSET_VERSION="${ASSET_VERSION%-${SUFFIX}.img.zip}"
115-
116-
echo "=== $SUFFIX: $NAME (version $ASSET_VERSION, $ASSET_DATE) ==="
117-
ZIP="work/$NAME"
118-
curl -fL --retry 3 -o "$ZIP" "$URL"
119-
120-
ZIP_SIZE=$(stat -c%s "$ZIP")
121-
ZIP_SHA=$(sha256sum "$ZIP" | awk '{print $1}')
122-
123-
IMG_DIR="work/extract-$SUFFIX"
124-
rm -rf "$IMG_DIR"
125-
mkdir -p "$IMG_DIR"
126-
unzip -q "$ZIP" -d "$IMG_DIR"
127-
IMG_FILE=$(find "$IMG_DIR" -maxdepth 1 -name '*.img' | head -n1)
128-
if [ -z "$IMG_FILE" ]; then
129-
echo "::error::No .img found inside $NAME" >&2
130-
exit 1
131-
fi
132-
133-
IMG_SIZE=$(stat -c%s "$IMG_FILE")
134-
IMG_SHA=$(sha256sum "$IMG_FILE" | awk '{print $1}')
135-
136-
# Free the extracted .img right away -- these are multi-GB and
137-
# four platforms in a row will exhaust the runner's disk otherwise.
138-
rm -rf "$IMG_DIR"
139-
rm -f "$ZIP"
140-
141-
{
142-
echo "${SUFFIX}_url=$URL"
143-
echo "${SUFFIX}_version=$ASSET_VERSION"
144-
echo "${SUFFIX}_release_date=$ASSET_DATE"
145-
echo "${SUFFIX}_image_download_size=$ZIP_SIZE"
146-
echo "${SUFFIX}_image_download_sha256=$ZIP_SHA"
147-
echo "${SUFFIX}_extract_size=$IMG_SIZE"
148-
echo "${SUFFIX}_extract_sha256=$IMG_SHA"
149-
} >> "$GITHUB_OUTPUT"
150-
done
116+
117+
MATCH=$(jq -c --arg s "$SUFFIX" \
118+
'[.assets[] | select(.name | test("-" + $s + "\\.img\\.zip$"))] | sort_by(.updated_at) | last // empty' \
119+
release.json)
120+
121+
if [ -z "$MATCH" ]; then
122+
echo "No .img.zip asset for platform $SUFFIX in this release -- skipping."
123+
exit 0
124+
fi
125+
126+
NAME=$(jq -r '.name' <<<"$MATCH")
127+
URL=$(jq -r '.browser_download_url' <<<"$MATCH")
128+
# Per-asset, not per-release: this repo re-uploads patch bumps
129+
# (9.5, 9.5.1, ..., 9.5.3) onto the SAME release/tag over time,
130+
# so the release object's own tag_name/published_at are stale
131+
# for anything past the first upload (see resolve job's note).
132+
ASSET_DATE=$(jq -r '(.updated_at // .created_at)' <<<"$MATCH" | cut -d'T' -f1)
133+
# Asset filenames are FPP-v<VERSION>-<SUFFIX>.img.zip (see
134+
# SD/build-image-pi.sh's OUT_IMG); strip the fixed prefix/suffix
135+
# to recover VERSION even though it can differ from the release
136+
# tag_name (FPP-v9.5.3-Pi.img.zip on tag "9.5").
137+
ASSET_VERSION="${NAME#FPP-v}"
138+
ASSET_VERSION="${ASSET_VERSION%-${SUFFIX}.img.zip}"
139+
140+
echo "=== $SUFFIX: $NAME (version $ASSET_VERSION, $ASSET_DATE) ==="
141+
ZIP="work/$NAME"
142+
curl -fL --retry 3 -o "$ZIP" "$URL"
143+
144+
ZIP_SIZE=$(stat -c%s "$ZIP")
145+
ZIP_SHA=$(sha256sum "$ZIP" | awk '{print $1}')
146+
147+
IMG_DIR="work/extract-$SUFFIX"
148+
rm -rf "$IMG_DIR"
149+
mkdir -p "$IMG_DIR"
150+
unzip -q "$ZIP" -d "$IMG_DIR"
151+
IMG_FILE=$(find "$IMG_DIR" -maxdepth 1 -name '*.img' | head -n1)
152+
if [ -z "$IMG_FILE" ]; then
153+
echo "::error::No .img found inside $NAME" >&2
154+
exit 1
155+
fi
156+
157+
IMG_SIZE=$(stat -c%s "$IMG_FILE")
158+
IMG_SHA=$(sha256sum "$IMG_FILE" | awk '{print $1}')
159+
160+
# Free the extracted .img right away -- these are multi-GB and
161+
# each matrix job only has one platform's worth of disk headroom.
162+
rm -rf "$IMG_DIR"
163+
rm -f "$ZIP"
164+
165+
jq -n \
166+
--arg url "$URL" \
167+
--arg version "$ASSET_VERSION" \
168+
--arg release_date "$ASSET_DATE" \
169+
--argjson image_download_size "$ZIP_SIZE" \
170+
--arg image_download_sha256 "$ZIP_SHA" \
171+
--argjson extract_size "$IMG_SIZE" \
172+
--arg extract_sha256 "$IMG_SHA" \
173+
'{url: $url, version: $version, release_date: $release_date,
174+
image_download_size: $image_download_size, image_download_sha256: $image_download_sha256,
175+
extract_size: $extract_size, extract_sha256: $extract_sha256}' \
176+
> "hashes-${SUFFIX}.json"
177+
178+
- name: Upload hashes
179+
uses: actions/upload-artifact@v4
180+
if: hashFiles(format('hashes-{0}.json', matrix.suffix)) != ''
181+
with:
182+
name: hashes-${{ matrix.suffix }}
183+
path: hashes-${{ matrix.suffix }}.json
184+
retention-days: 1
185+
186+
update:
187+
needs: [resolve, hashes]
188+
runs-on: ubuntu-latest
189+
env:
190+
GH_TOKEN: ${{ github.token }}
191+
TAG: ${{ needs.resolve.outputs.tag }}
192+
steps:
193+
- name: Checkout master
194+
uses: actions/checkout@v5
195+
with:
196+
ref: master
197+
198+
- name: Download per-platform hashes
199+
uses: actions/download-artifact@v4
200+
with:
201+
pattern: hashes-*
202+
path: hashes
203+
merge-multiple: false
151204

152205
- name: Update JSON
153206
id: update
154-
env:
155-
TAG: ${{ steps.release.outputs.tag }}
156-
Pi_url: ${{ steps.hashes.outputs.Pi_url }}
157-
Pi_version: ${{ steps.hashes.outputs.Pi_version }}
158-
Pi_release_date: ${{ steps.hashes.outputs.Pi_release_date }}
159-
Pi_image_download_size: ${{ steps.hashes.outputs.Pi_image_download_size }}
160-
Pi_image_download_sha256: ${{ steps.hashes.outputs.Pi_image_download_sha256 }}
161-
Pi_extract_size: ${{ steps.hashes.outputs.Pi_extract_size }}
162-
Pi_extract_sha256: ${{ steps.hashes.outputs.Pi_extract_sha256 }}
163-
Pi64_url: ${{ steps.hashes.outputs.Pi64_url }}
164-
Pi64_version: ${{ steps.hashes.outputs.Pi64_version }}
165-
Pi64_release_date: ${{ steps.hashes.outputs.Pi64_release_date }}
166-
Pi64_image_download_size: ${{ steps.hashes.outputs.Pi64_image_download_size }}
167-
Pi64_image_download_sha256: ${{ steps.hashes.outputs.Pi64_image_download_sha256 }}
168-
Pi64_extract_size: ${{ steps.hashes.outputs.Pi64_extract_size }}
169-
Pi64_extract_sha256: ${{ steps.hashes.outputs.Pi64_extract_sha256 }}
170-
BB64_url: ${{ steps.hashes.outputs.BB64_url }}
171-
BB64_version: ${{ steps.hashes.outputs.BB64_version }}
172-
BB64_release_date: ${{ steps.hashes.outputs.BB64_release_date }}
173-
BB64_image_download_size: ${{ steps.hashes.outputs.BB64_image_download_size }}
174-
BB64_image_download_sha256: ${{ steps.hashes.outputs.BB64_image_download_sha256 }}
175-
BB64_extract_size: ${{ steps.hashes.outputs.BB64_extract_size }}
176-
BB64_extract_sha256: ${{ steps.hashes.outputs.BB64_extract_sha256 }}
177-
BBB_url: ${{ steps.hashes.outputs.BBB_url }}
178-
BBB_version: ${{ steps.hashes.outputs.BBB_version }}
179-
BBB_release_date: ${{ steps.hashes.outputs.BBB_release_date }}
180-
BBB_image_download_size: ${{ steps.hashes.outputs.BBB_image_download_size }}
181-
BBB_image_download_sha256: ${{ steps.hashes.outputs.BBB_image_download_sha256 }}
182-
BBB_extract_size: ${{ steps.hashes.outputs.BBB_extract_size }}
183-
BBB_extract_sha256: ${{ steps.hashes.outputs.BBB_extract_sha256 }}
184207
run: |
185208
python3 <<'PY'
209+
import glob
186210
import json
187211
import os
188212
import re
@@ -243,19 +267,25 @@ jobs:
243267
m = name_re.match(o.get("name", ""))
244268
return channel_of(m.group(1)) if m else "stable"
245269
270+
# Each parallel "hashes" matrix job dropped hashes-<SUFFIX>.json
271+
# (absent if that platform had no asset in this release).
272+
per_platform = {}
273+
for f in glob.glob("hashes/hashes-*/hashes-*.json"):
274+
suffix = re.match(r"hashes-(.+)\.json$", os.path.basename(f)).group(1)
275+
with open(f) as fh:
276+
per_platform[suffix] = json.load(fh)
277+
246278
added, updated, removed_dupes, superseded, skipped = [], [], [], [], []
247279
channels_seen = set()
248280
249281
for suffix in PLATFORM_SUFFIXES:
250-
url = os.environ.get(f"{suffix}_url", "")
251-
if not url:
282+
fields = per_platform.get(suffix)
283+
if not fields:
252284
continue # no asset for this platform in the release
253285
254-
# Per-platform, not per-release: see the note by the "Resolve
255-
# release" step -- a single release/tag can carry several
256-
# differently-versioned assets uploaded over time.
257-
version = os.environ[f"{suffix}_version"]
258-
release_date = os.environ[f"{suffix}_release_date"]
286+
url = fields["url"]
287+
version = fields["version"]
288+
release_date = fields["release_date"]
259289
new_channel = channel_of(version)
260290
new_family = family_of(version)
261291
channels_seen.add(new_channel)
@@ -324,10 +354,10 @@ jobs:
324354
label = m.group(2) if m else suffix
325355
entry["name"] = f"FPP v{version} {label}"
326356
entry["url"] = url
327-
entry["image_download_size"] = int(os.environ[f"{suffix}_image_download_size"])
328-
entry["image_download_sha256"] = os.environ[f"{suffix}_image_download_sha256"]
329-
entry["extract_size"] = int(os.environ[f"{suffix}_extract_size"])
330-
entry["extract_sha256"] = os.environ[f"{suffix}_extract_sha256"]
357+
entry["image_download_size"] = int(fields["image_download_size"])
358+
entry["image_download_sha256"] = fields["image_download_sha256"]
359+
entry["extract_size"] = int(fields["extract_size"])
360+
entry["extract_sha256"] = fields["extract_sha256"]
331361
entry["release_date"] = release_date
332362
333363
with open(path, "w") as f:
@@ -356,7 +386,6 @@ jobs:
356386
- name: Open PR
357387
if: ${{ inputs.dry_run != true && steps.update.outputs.changed != '' }}
358388
env:
359-
TAG: ${{ steps.release.outputs.tag }}
360389
CHANNEL: ${{ steps.update.outputs.channel }}
361390
ADDED: ${{ steps.update.outputs.added }}
362391
UPDATED: ${{ steps.update.outputs.updated }}

0 commit comments

Comments
 (0)