CI deps image #15
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI deps image | |
| # Builds the prebuilt apt .deb bundles that the make-check family (the | |
| # -minimal tags), the interop workflows (the -full tags, a superset), the | |
| # membrowse embedded targets (the -embedded tag - the big ARM cross-toolchain) | |
| # and the linux kernel-module builds (the -linuxkm tag - kernel headers) | |
| # install offline (see .github/actions/install-apt-deps, input | |
| # ghcr-debs-tag). Each bundle holds the .debs for a package list in | |
| # .github/ci-deps/ - every package plus the dependencies not already on the | |
| # matching runner image, so it is tied to that runner rather than being a | |
| # portable, self-contained closure - published to | |
| # ghcr.io/<owner>/wolfssl-ci-debs:<tag>. | |
| # | |
| # Why: the apt mirror times out often enough to break PR CI. Resolving the | |
| # closure ONCE here (on master, where a slow mirror only delays this job and | |
| # is retried hard) and pulling it from ghcr on every PR keeps apt off the PR | |
| # critical path entirely. ghcr storage/bandwidth is free for public images | |
| # and is a separate pool from the 10 GB Actions cache. | |
| # | |
| # ONE-TIME SETUP: after the first successful run, make the package | |
| # `wolfssl-ci-debs` PUBLIC (repo/org > Packages > Package settings > | |
| # Change visibility). Anonymous `docker pull` then works from fork PRs too; | |
| # until then install-apt-deps simply falls back to apt (no breakage). | |
| on: | |
| schedule: | |
| # Weekly (Saturday) - the static bundles (-minimal/-full/-embedded). | |
| # Refreshes them so they track base-image security updates. A mid-week | |
| # package-list change waits for Saturday (or run this manually via | |
| # workflow_dispatch); until then the offline install (a single | |
| # --no-download install of the whole set) fails if any requested package | |
| # is missing from the bundle, and install-apt-deps falls back to apt. | |
| - cron: '0 2 * * 6' | |
| # Daily - the kernel-tracking -linuxkm bundle only. linux-headers-$(uname | |
| # -r) pins to the runner's running kernel (changes ~monthly); the linuxkm | |
| # job rebuilds solely when uname -r differs from the published bundle, a | |
| # cheap no-op otherwise. A mismatch mid-rollout just falls back to apt. | |
| - cron: '0 3 * * *' | |
| workflow_dispatch: | |
| concurrency: | |
| group: ci-deps-image-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| packages: write | |
| jobs: | |
| build: | |
| name: build ${{ matrix.tag }} | |
| # Static bundles: weekly cron or manual dispatch. Skip the daily cron, | |
| # which exists only to refresh the kernel-tracking -linuxkm bundle below. | |
| if: >- | |
| github.repository_owner == 'wolfssl' && | |
| (github.event_name != 'schedule' || github.event.schedule == '0 2 * * 6') | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # The .debs must be downloaded on the same Ubuntu version that | |
| # consumes them, so the runner matches the tag. -minimal is the | |
| # make-check family's packages (small, pulled on every PR); | |
| # -full adds the interop workflows' packages (a superset). | |
| - runner: ubuntu-24.04 | |
| tag: ubuntu-24.04-minimal | |
| - runner: ubuntu-24.04 | |
| tag: ubuntu-24.04-full | |
| # membrowse embedded targets' ARM cross-toolchain (~0.5 GB). Its own | |
| # tag so it does not bloat the -full pull for the interop workflows. | |
| - runner: ubuntu-24.04 | |
| tag: ubuntu-24.04-embedded | |
| - runner: ubuntu-22.04 | |
| tag: ubuntu-22.04-minimal | |
| - runner: ubuntu-22.04 | |
| tag: ubuntu-22.04-full | |
| runs-on: ${{ matrix.runner }} | |
| # Backstop only: the download step kills and retries stalled apt connections, | |
| # but cap the job so a pathological mirror cannot hang a runner. -full | |
| # normally finishes in a few minutes. | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Resolve and download the .deb closure | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| LIST=".github/ci-deps/packages-${{ matrix.tag }}.txt" | |
| mapfile -t PKGS < <(grep -vE '^[[:space:]]*#|^[[:space:]]*$' "$LIST") | |
| echo "Packages (${#PKGS[@]}): ${PKGS[*]}" | |
| export DEBIAN_FRONTEND=noninteractive | |
| rm -rf debs && mkdir -p debs | |
| sudo apt-get clean | |
| # A single stalled mirror connection once hung -full for ~20 min (it | |
| # normally finishes in a few). retry() only re-runs on a non-zero exit, | |
| # so a hang never tripped it. Defend in depth: apt drops a stalled | |
| # connection after 30s and retries it (Acquire timeouts), `timeout` | |
| # hard-kills a wedged apt-get, then retry() re-runs from scratch. | |
| APT_OPTS=(-o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30) | |
| retry() { local i; for i in 1 2 3 4 5; do "$@" && return 0; sleep $((2**i)); done; "$@"; } | |
| retry sudo timeout -k 10 120 apt-get "${APT_OPTS[@]}" update -q | |
| # Download each package's closure independently (requested package + | |
| # any dependency not already installed) without installing. Per | |
| # package, not one resolve of the whole list, so one unbundleable | |
| # package - e.g. a conflict in the big -full union - cannot abort the | |
| # rest; install-apt-deps falls back to apt for anything missing. | |
| skipped=0 | |
| for pkg in "${PKGS[@]}"; do | |
| retry sudo timeout -k 10 300 apt-get "${APT_OPTS[@]}" install -y --download-only "$pkg" \ | |
| || { echo "::warning::could not download $pkg"; skipped=$((skipped+1)); } | |
| done | |
| sudo cp /var/cache/apt/archives/*.deb debs/ 2>/dev/null || true | |
| echo "Bundled $(ls debs/*.deb 2>/dev/null | wc -l) .deb files ($(du -sh debs | cut -f1)); ${skipped} skipped" | |
| test -n "$(ls debs/*.deb 2>/dev/null)" # fail if nothing was bundled | |
| - name: Build bundle image | |
| shell: bash | |
| run: | | |
| # Tiny busybox base so the consumer can `docker create`/`docker cp` | |
| # the .debs out; the base size is negligible next to the .debs. | |
| printf 'FROM busybox\nCOPY debs /debs\n' > Dockerfile.debs | |
| docker build -f Dockerfile.debs -t bundle . | |
| - name: Log in to ghcr | |
| shell: bash | |
| run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Push to ghcr | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') | |
| IMG="ghcr.io/$OWNER/wolfssl-ci-debs" | |
| # One mutable tag per bundle variant; each run overwrites it, so the | |
| # package keeps exactly one version per variant (no dated duplicates). | |
| docker tag bundle "$IMG:${{ matrix.tag }}" | |
| docker push "$IMG:${{ matrix.tag }}" | |
| echo "Pushed $IMG:${{ matrix.tag }}" | |
| # Kernel-tracking bundle for the linux kernel-module builds (linuxkm.yml and | |
| # the membrowse linuxkm targets). linux-headers-$(uname -r) pins to the | |
| # runner's running kernel, so this runs daily but rebuilds only when the | |
| # kernel changed since the published bundle (the image carries the kernel as | |
| # a label). A mismatch - e.g. during a gradual runner-image rollout - just | |
| # makes install-apt-deps fall back to apt. | |
| linuxkm: | |
| name: build ubuntu-24.04-linuxkm | |
| if: github.repository_owner == 'wolfssl' | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Log in to ghcr | |
| shell: bash | |
| run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin | |
| - name: Decide whether the published bundle already matches this kernel | |
| id: check | |
| shell: bash | |
| run: | | |
| set -uo pipefail | |
| OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') | |
| IMG="ghcr.io/$OWNER/wolfssl-ci-debs:ubuntu-24.04-linuxkm" | |
| K=$(uname -r) | |
| echo "kernel=$K" >> "$GITHUB_OUTPUT" | |
| echo "runner kernel: $K" | |
| have="" | |
| if docker pull -q "$IMG" >/dev/null 2>&1; then | |
| have=$(docker inspect --format '{{ index .Config.Labels "kernel" }}' "$IMG" 2>/dev/null || true) | |
| fi | |
| echo "published bundle kernel: ${have:-<none>}" | |
| if [ "$have" = "$K" ]; then | |
| echo "rebuild=false" >> "$GITHUB_OUTPUT" | |
| echo "Bundle already current for $K; nothing to do." | |
| else | |
| echo "rebuild=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Resolve and download the .deb closure | |
| if: steps.check.outputs.rebuild == 'true' | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| K="${{ steps.check.outputs.kernel }}" | |
| # linuxkm.yml installs only the headers; the membrowse linuxkm targets | |
| # also need the build toolchain. Bundle the union - each consumer | |
| # installs its own subset offline. | |
| PKGS=(build-essential autoconf automake libtool "linux-headers-$K") | |
| echo "Packages: ${PKGS[*]}" | |
| export DEBIAN_FRONTEND=noninteractive | |
| rm -rf debs && mkdir -p debs | |
| sudo apt-get clean | |
| retry() { local i; for i in 1 2 3 4 5; do "$@" && return 0; sleep $((2**i)); done; "$@"; } | |
| retry sudo apt-get update -q | |
| # The whole set is required and this bundle is small, so resolve it as | |
| # one closure and let any download failure fail the job. We push only | |
| # on success, so a transient mirror error keeps the last good bundle | |
| # rather than publishing a partial one - which the kernel-label skip | |
| # would then pin in place until the kernel next changes (~monthly). | |
| retry sudo apt-get install -y --download-only "${PKGS[@]}" | |
| sudo cp /var/cache/apt/archives/*.deb debs/ 2>/dev/null || true | |
| echo "Bundled $(ls debs/*.deb 2>/dev/null | wc -l) .deb files" | |
| test -n "$(ls debs/*.deb 2>/dev/null)" # headers are never preinstalled | |
| - name: Build and push bundle (labelled with the kernel) | |
| if: steps.check.outputs.rebuild == 'true' | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| K="${{ steps.check.outputs.kernel }}" | |
| OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') | |
| IMG="ghcr.io/$OWNER/wolfssl-ci-debs:ubuntu-24.04-linuxkm" | |
| printf 'FROM busybox\nCOPY debs /debs\nLABEL kernel=%s\n' "$K" > Dockerfile.debs | |
| docker build -f Dockerfile.debs -t "$IMG" . | |
| docker push "$IMG" | |
| echo "Pushed $IMG (kernel $K)" |