perf(ci): cache slow build steps and skip redundant work #949
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: "Build Packages and tests on Droplets" | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| # The branches below must be a subset of the branches above | |
| branches: [main] | |
| # This action build the package for Ubuntu and Debian | |
| # Then run e2e integration tests on Digital Ocean droplets (on demand VM) | |
| # Cancel superseded runs when new commits are pushed to a PR branch. | |
| # This also releases the global droplet concurrency groups earlier. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| build_deb: | |
| name: "Build ${{ matrix.os }} Package" | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ["debian-12", "debian-13", "ubuntu-22.04", "ubuntu-24.04"] | |
| include: | |
| - os: "debian-12" | |
| make_target: "all-podman-debian-12" | |
| artifact_name: "aleph-vm.debian-12.deb" | |
| - os: "debian-13" | |
| make_target: "all-podman-debian-13" | |
| artifact_name: "aleph-vm.debian-13.deb" | |
| - os: "ubuntu-22.04" | |
| make_target: "all-podman-ubuntu-2204" | |
| artifact_name: "aleph-vm.ubuntu-22.04.deb" | |
| - os: "ubuntu-24.04" | |
| make_target: "all-podman-ubuntu-2404" | |
| artifact_name: "aleph-vm.ubuntu-24.04.deb" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| # Fetch the whole history for all tags and branches (required for aleph.__version__) | |
| fetch-depth: 0 | |
| - name: Initialize git submodules | |
| run: git submodule init | |
| # sevctl is built with `cargo install` at a rev pinned in packaging/Makefile, | |
| # which takes ~2 minutes on every build. `target/bin/sevctl` is a file target | |
| # in the Makefile, so restoring the binary from cache skips the build entirely. | |
| # The key includes the Makefile (pins the sevctl rev) and the dockerfile | |
| # (pins the toolchain used to build it). | |
| - name: Cache the sevctl binary | |
| uses: actions/cache@v4 | |
| with: | |
| path: packaging/target/bin | |
| key: sevctl-${{ matrix.os }}-${{ hashFiles('packaging/Makefile', format('packaging/{0}.dockerfile', | |
| matrix.os)) }} | |
| - run: | | |
| cd packaging && make ${{ matrix.make_target }} && cd .. | |
| ls packaging/target | |
| - name: Ensure that the relevant files are present in the package | |
| run: | | |
| dpkg --contents packaging/target/${{ matrix.artifact_name }} | grep /opt/firecracker/firecracker | |
| dpkg --contents packaging/target/${{ matrix.artifact_name }} | grep /opt/firecracker/jailer | |
| dpkg --contents packaging/target/${{ matrix.artifact_name }} | grep /opt/firecracker/vmlinux.bin | |
| dpkg --contents packaging/target/${{ matrix.artifact_name }} | grep /opt/sevctl | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.artifact_name }} | |
| path: packaging/target/${{ matrix.artifact_name }} | |
| build_rootfs: | |
| name: "Build runtime aleph-${{ matrix.os }}-python" | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| os: ["debian-12"] | |
| include: | |
| - os: "debian-12" | |
| artifact_name: "aleph-debian-12-python.squashfs" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Workaround github issue https://github.com/actions/runner-images/issues/7192 | |
| run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc | |
| # The runtime takes ~2 minutes to debootstrap and rarely changes. Cache it on | |
| # the content of its build inputs, with a monthly stamp so the packages | |
| # pulled by debootstrap and pip do not become indefinitely stale. | |
| # The key is computed in shell because hashFiles() is re-evaluated in the | |
| # cache post step, after the build filled the directory with the | |
| # debootstrapped tree (hashing it times out). | |
| - name: Compute runtime cache key | |
| id: cache-key | |
| run: | | |
| input_hash=$(cat runtimes/aleph-${{ matrix.os }}-python/*.sh runtimes/aleph-${{ matrix.os }}-python/*.py runtimes/aleph-${{ matrix.os }}-python/*.html | sha256sum | cut -c -16) | |
| echo "key=runtime-aleph-${{ matrix.os }}-python-$(date +%Y-%m)-${input_hash}" >> "$GITHUB_OUTPUT" | |
| - name: Cache the built runtime | |
| id: cache-runtime | |
| uses: actions/cache@v4 | |
| with: | |
| path: runtimes/aleph-${{ matrix.os }}-python/rootfs.squashfs | |
| key: ${{ steps.cache-key.outputs.key }} | |
| - name: Install dep and build | |
| if: steps.cache-runtime.outputs.cache-hit != 'true' | |
| run: | | |
| sudo apt update | |
| sudo apt install -y debootstrap | |
| cd runtimes/aleph-${{ matrix.os }}-python && sudo ./create_disk_image.sh && cd ../.. | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.artifact_name }} | |
| path: runtimes/aleph-${{ matrix.os }}-python/rootfs.squashfs | |
| build_example_venv_volume: | |
| name: "Build example squashfs volume using Docker" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - run: | | |
| docker build -t aleph-vm-build-squashfs -f examples/volumes/Dockerfile examples/volumes | |
| docker run --rm -v "$(pwd)":/mnt aleph-vm-build-squashfs | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: example-volume-venv.squashfs | |
| path: volume-venv.squashfs | |
| run_on_droplet: | |
| # These are end-to-end tests running on ephemeral DigitalOcean "Droplet" virtual machines | |
| # with the different operating systems that are supported. | |
| # | |
| # The main focus of these tests is to ensure that the packaging works on all supported platforms | |
| # and to ensure the compatibility of dependencies (system and vendored) across these platforms. | |
| # | |
| # All supported runtimes are tested sequentially on the same droplet to avoid | |
| # the overhead of provisioning separate VMs (the setup is the slow part). | |
| name: "Test Droplet with ${{ matrix.os_config.os_name }}" | |
| runs-on: ubuntu-latest | |
| concurrency: "${{ matrix.os_config.concurrency_group }}" | |
| timeout-minutes: 20 | |
| needs: build_deb | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Check compatibility with all supported OSes. | |
| os_config: | |
| - os_name: "Debian 12" | |
| os_image: "debian-12-x64" | |
| alias: "debian-12" | |
| package_build_command: "all-podman-debian-12" | |
| package_name: "aleph-vm.debian-12.deb" | |
| concurrency_group: "droplet-aleph-vm-debian-12" | |
| - os_name: "Debian 13" | |
| os_image: "debian-13-x64" | |
| alias: "debian-13" | |
| package_build_command: "all-podman-debian-13" | |
| package_name: "aleph-vm.debian-13.deb" | |
| concurrency_group: "droplet-aleph-vm-debian-13" | |
| - os_name: "Ubuntu 22.04" | |
| os_image: "ubuntu-22-04-x64" | |
| alias: "ubuntu-22-04" | |
| package_build_command: "all-podman-ubuntu-2204" | |
| package_name: "aleph-vm.ubuntu-22.04.deb" | |
| concurrency_group: "droplet-aleph-vm-ubuntu-22-04" | |
| - os_name: "Ubuntu 24.04" | |
| os_image: "ubuntu-24-04-x64" | |
| alias: "ubuntu-24-04" | |
| package_build_command: "all-podman-ubuntu-2404" | |
| package_name: "aleph-vm.ubuntu-24.04.deb" | |
| concurrency_group: "droplet-aleph-vm-ubuntu-24-04" | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install doctl | |
| uses: digitalocean/action-doctl@v2 | |
| with: | |
| token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN_V2 }} | |
| - name: Setup SSH private key | |
| run: | | |
| mkdir ~/.ssh | |
| echo $DIGITALOCEAN_SSH_PRIVATE_KEY | base64 --decode > ~/.ssh/id_ed25519 | |
| chmod 0700 ~/.ssh | |
| chmod 0600 ~/.ssh/id_ed25519 | |
| env: | |
| DIGITALOCEAN_SSH_PRIVATE_KEY: ${{ secrets.DIGITALOCEAN_SSH_PRIVATE_KEY }} | |
| - name: Create the Droplet | |
| run: | | |
| doctl compute droplet create \ | |
| --image ${{ matrix.os_config.os_image }} \ | |
| --size s-2vcpu-4gb-amd \ | |
| --region ams3 \ | |
| --vpc-uuid 5976b7bd-4417-49e8-8522-672aaa920c30 \ | |
| --enable-ipv6 \ | |
| --ssh-keys ab:2b:25:16:46:6f:25:d0:80:63:e5:be:67:04:cb:64 \ | |
| aleph-vm-ci-${{ matrix.os_config.alias }} | |
| - uses: actions/download-artifact@v4 | |
| name: "Download the package from artifacts." | |
| with: | |
| name: ${{ matrix.os_config.package_name }} | |
| path: packaging/target/ | |
| - name: Get droplet ip and export it in env | |
| # the until loop wait till the network is available. | |
| run: | | |
| until (doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py); do sleep 1; done | |
| echo "DROPLET_IPV4=$(doctl compute droplet get aleph-vm-ci-${{ matrix.os_config.alias }} --output json | ./.github/scripts/extract_droplet_ipv4.py)" >> "$GITHUB_ENV" | |
| - name: Wait for the system to setup and boot | |
| id: system-booted | |
| run: | | |
| until ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts 2>/dev/null; do sleep 5; done | |
| timeout-minutes: 5 | |
| - name: Install Aleph-VM on the Droplet | |
| run: | | |
| set -x | |
| ## Build configuration file and copy it on Droplet | |
| echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> supervisor.env | |
| echo ALEPH_VM_ALLOCATION_TOKEN_HASH=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 >> supervisor.env | |
| echo ALEPH_VM_SENTRY_DSN=${{ secrets.SENTRY_DSN }} >> supervisor.env | |
| # Pin public resolvers for guest VMs: DNS auto-detection on the droplet | |
| # picks up DigitalOcean's VPC-internal resolver (e.g. 10.110.15.254), | |
| # which is unreachable from inside the VMs and breaks all guest DNS. | |
| echo 'ALEPH_VM_DNS_NAMESERVERS=["1.1.1.1","8.8.8.8"]' >> supervisor.env | |
| ssh root@${DROPLET_IPV4} mkdir -p /etc/aleph-vm/ | |
| scp supervisor.env root@${DROPLET_IPV4}:/etc/aleph-vm/supervisor.env | |
| scp packaging/target/${{ matrix.os_config.package_name }} root@${DROPLET_IPV4}:/opt | |
| # Wait a few seconds for DigitalOcean to setup the Droplet using apt, which conflicts with our comands: | |
| sleep 5 | |
| # Wait for /var/lib/apt/lists/lock to be unlocked on the remote host via SSH. | |
| while ssh root@${DROPLET_IPV4} lsof /var/lib/apt/lists/lock; do sleep 1; done | |
| ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o Dpkg::Progress-Fancy=0 -o DPkg::Lock::Timeout=-1 update" | |
| # A full system upgrade takes ~1 minute. Only run it on pushes to main | |
| # to catch incompatibilities with the latest distro packages, and skip | |
| # it on pull requests to keep iteration fast. | |
| if [ "${{ github.event_name }}" != "pull_request" ]; then | |
| ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o Dpkg::Progress-Fancy=0 -o DPkg::Lock::Timeout=-1 upgrade -y" | |
| fi | |
| ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o Dpkg::Progress-Fancy=0 -o DPkg::Lock::Timeout=-1 install -y docker.io apparmor-profiles" | |
| ssh root@${DROPLET_IPV4} "docker pull ghcr.io/aleph-im/vm-connector:alpha" | |
| ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector ghcr.io/aleph-im/vm-connector:alpha" | |
| # "--force-confold" keeps existing config files during package install/upgrade, avoiding prompts. | |
| ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o Dpkg::Progress-Fancy=0 -o DPkg::Lock::Timeout=-1 -o Dpkg::Options::="--force-confold" install -y /opt/${{ matrix.os_config.package_name }}" | |
| - name: Wait for supervisor to be ready | |
| run: | | |
| echo "Waiting for aleph-vm-supervisor to listen on port 4020..." | |
| timeout 90 bash -c 'until ssh root@${DROPLET_IPV4} "systemctl is-active --quiet aleph-vm-supervisor && ss -tlnp | grep -q :4020\ " 2>/dev/null; do sleep 2; done' | |
| echo "Supervisor is active and listening" | |
| - name: Dump supervisor logs on failure | |
| if: failure() | |
| run: | | |
| ssh root@${DROPLET_IPV4} "systemctl status aleph-vm-supervisor --no-pager" || true | |
| ssh root@${DROPLET_IPV4} "journalctl -u aleph-vm-supervisor -n 100 --no-pager" || true | |
| - name: "Test runtime: Debian 12, SDK 0.9.0" | |
| run: ./.github/scripts/test_runtime_on_droplet.sh "${DROPLET_IPV4}" "63faf8b5db1cf8d965e6a464a0cb8062af8e7df131729e48738342d956f29ace" | |
| - name: "Test runtime: Debian 12, pydantic V2, SDK 2.0.5" | |
| run: ./.github/scripts/test_runtime_on_droplet.sh "${DROPLET_IPV4}" "d2b74aa29898457bde0560e47f7cdd4e77287e9f1f7a1456161d2fd7d5c855d7" | |
| - name: Fetch system usage endpoint | |
| run: | | |
| curl -X GET -H "Content-Type: application/json" \ | |
| "http://${DROPLET_IPV4}:4020/about/usage/system" | |
| - name: Run the sevctl command to ensure it's properly packaged and working | |
| run: | | |
| ssh root@${DROPLET_IPV4} "/opt/sevctl --version" | |
| - name: Export aleph logs | |
| continue-on-error: true | |
| if: ${{ !cancelled() && steps.system-booted.outcome == 'success'}} | |
| run: | | |
| ssh root@${DROPLET_IPV4} "journalctl -u aleph-vm-supervisor" | |
| - name: Cleanup | |
| if: always() | |
| run: |- | |
| DROPLET_IDS=$(doctl compute droplet list --format "ID,Name" --no-header | grep "aleph-vm-ci-${{ matrix.os_config.alias }}" | awk '{print $1}') | |
| for DROPLET_ID in $DROPLET_IDS; do | |
| echo "Deleting droplet with ID: $DROPLET_ID" | |
| doctl compute droplet delete --force $DROPLET_ID | |
| done | |
| run_new_runtime_debian_12: | |
| name: "Test new runtime on Droplet with Debian 12" | |
| # Test by building a version of the runtimes and diagnostic program from the source. | |
| # (other tests use the version deployed in the aleph cloud) | |
| runs-on: ubuntu-latest | |
| concurrency: droplet-aleph-vm-runtime | |
| timeout-minutes: 10 | |
| needs: [build_deb, build_rootfs, build_example_venv_volume] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| # Fetch the whole history for all tags and branches (required for aleph.__version__) | |
| fetch-depth: 0 | |
| - name: Install doctl | |
| uses: digitalocean/action-doctl@v2 | |
| with: | |
| token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN_V2 }} | |
| - name: Setup SSH private key | |
| run: | | |
| mkdir ~/.ssh | |
| echo $DIGITALOCEAN_SSH_PRIVATE_KEY | base64 --decode > ~/.ssh/id_ed25519 | |
| chmod 0700 ~/.ssh | |
| chmod 0600 ~/.ssh/id_ed25519 | |
| env: | |
| DIGITALOCEAN_SSH_PRIVATE_KEY: ${{ secrets.DIGITALOCEAN_SSH_PRIVATE_KEY }} | |
| - name: Create the Droplet | |
| run: | | |
| doctl compute droplet create \ | |
| --image debian-12-x64 \ | |
| --size s-2vcpu-4gb-amd \ | |
| --region ams3 \ | |
| --vpc-uuid 5976b7bd-4417-49e8-8522-672aaa920c30 \ | |
| --enable-ipv6 \ | |
| --ssh-keys ab:2b:25:16:46:6f:25:d0:80:63:e5:be:67:04:cb:64 \ | |
| aleph-vm-ci-runtime | |
| # Reuse the runtime built by the build_rootfs job instead of spending | |
| # ~2 minutes rebuilding it here with debootstrap. | |
| - uses: actions/download-artifact@v4 | |
| name: "Download the runtime from artifacts." | |
| with: | |
| name: "aleph-debian-12-python.squashfs" | |
| path: runtimes/aleph-debian-12-python/ | |
| - uses: actions/download-artifact@v4 | |
| name: "Download the debian package from artifacts." | |
| with: | |
| name: "aleph-vm.debian-12.deb" | |
| path: packaging/target/ | |
| # The supervisor requires the example venv volume when running with fake data | |
| # (ALEPH_VM_FAKE_DATA_VOLUME, asserted at startup). It is built by the | |
| # build_example_venv_volume job; place it where `scp -pr ./examples` ships it. | |
| - uses: actions/download-artifact@v4 | |
| name: "Download the example venv volume from artifacts." | |
| with: | |
| name: "example-volume-venv.squashfs" | |
| path: examples/volumes/ | |
| # - name: Build Debian Package | |
| # run: | | |
| # cd packaging && make all-podman-debian-12 && cd .. | |
| # ls packaging/target | |
| - name: Get droplet ip and export it in env | |
| # the until loop wait till the network is available. | |
| run: | | |
| until (doctl compute droplet get aleph-vm-ci-runtime --output json | ./.github/scripts/extract_droplet_ipv4.py); do sleep 1; done | |
| echo "DROPLET_IPV4=$(doctl compute droplet get aleph-vm-ci-runtime --output json | ./.github/scripts/extract_droplet_ipv4.py)" >> "$GITHUB_ENV" | |
| - name: Wait for the system to setup and boot | |
| id: system-booted | |
| timeout-minutes: 3 | |
| run: | | |
| until ssh-keyscan -H ${DROPLET_IPV4} > ~/.ssh/known_hosts 2>/dev/null; do sleep 5; done | |
| - name: Copy the runtime to the system | |
| run: | | |
| scp runtimes/aleph-debian-12-python/rootfs.squashfs root@${DROPLET_IPV4}:/opt | |
| - name: Install Aleph-VM on the Droplet | |
| run: | | |
| ## Build configuration file and copy it on Droplet | |
| echo ALEPH_VM_SUPERVISOR_HOST=0.0.0.0 >> supervisor.env | |
| echo ALEPH_VM_FAKE_DATA_PROGRAM=/opt/examples/example_fastapi >> supervisor.env | |
| echo ALEPH_VM_FAKE_DATA_RUNTIME=/opt/rootfs.squashfs >> supervisor.env | |
| # Pin public resolvers for guest VMs: DNS auto-detection on the droplet | |
| # picks up DigitalOcean's VPC-internal resolver (e.g. 10.110.15.254), | |
| # which is unreachable from inside the VMs and breaks all guest DNS. | |
| echo 'ALEPH_VM_DNS_NAMESERVERS=["1.1.1.1","8.8.8.8"]' >> supervisor.env | |
| ssh root@${DROPLET_IPV4} mkdir -p /etc/aleph-vm/ | |
| scp supervisor.env root@${DROPLET_IPV4}:/etc/aleph-vm/supervisor.env | |
| scp packaging/target/aleph-vm.debian-12.deb root@${DROPLET_IPV4}:/opt | |
| ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o Dpkg::Progress-Fancy=0 -o DPkg::Lock::Timeout=-1 update" | |
| # A full system upgrade takes ~1 minute. Only run it on pushes to main | |
| # to catch incompatibilities with the latest distro packages, and skip | |
| # it on pull requests to keep iteration fast. | |
| if [ "${{ github.event_name }}" != "pull_request" ]; then | |
| ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o Dpkg::Progress-Fancy=0 -o DPkg::Lock::Timeout=-1 upgrade -y" | |
| fi | |
| ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o Dpkg::Progress-Fancy=0 -o DPkg::Lock::Timeout=-1 install -y docker.io apparmor-profiles" | |
| ssh root@${DROPLET_IPV4} "docker run -d -p 127.0.0.1:4021:4021/tcp --restart=always --name vm-connector alephim/vm-connector:alpha" | |
| # Copy our example VM on the server | |
| scp -pr ./examples root@${DROPLET_IPV4}:/opt/ | |
| ssh root@${DROPLET_IPV4} DEBIAN_FRONTEND=noninteractive "apt-get -o DPkg::Lock::Timeout=-1 -o Dpkg::Options::="--force-confold" install -y /opt/aleph-vm.debian-12.deb" | |
| - name: Call the runtime and example program on the Droplet | |
| run: | | |
| sleep 3 | |
| curl --retry 5 --retry-delay 10 --retry-connrefused --max-time 120 --fail "http://${DROPLET_IPV4}:4020/about/usage/system" | |
| curl --retry 5 --retry-delay 10 --retry-connrefused --max-time 120 --fail "http://${DROPLET_IPV4}:4020/status/check/fastapi" | |
| - name: Export aleph logs | |
| if: ${{ !cancelled() && steps.system-booted.outcome == 'success'}} | |
| run: | | |
| ssh root@${DROPLET_IPV4} "journalctl -u aleph-vm-supervisor" | |
| - name: Cleanup | |
| if: always() | |
| run: |- | |
| doctl compute droplet delete -f aleph-vm-ci-runtime |