Merge branch 'ci/install/batch' #75
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Install slices | |
| on: | |
| push: | |
| branches: | |
| - "main" | |
| paths: | |
| - ".github/scripts/install-slices/**" | |
| - ".github/workflows/install-slices.yaml" | |
| pull_request: | |
| branches: | |
| - "main" | |
| paths: | |
| - ".github/scripts/install-slices/**" | |
| - ".github/workflows/install-slices.yaml" | |
| schedule: | |
| # Run at 00:00 every day. | |
| # Ref: https://man7.org/linux/man-pages/man5/crontab.5.html | |
| - cron: "0 0 * * *" | |
| workflow_call: | |
| env: | |
| # Package architectures and chisel-releases branches to test on. | |
| ARCHES: '["amd64","arm64","armhf","i386","ppc64el","riscv64","s390x"]' | |
| RELEASES: | | |
| [ | |
| {"ref": "ubuntu-20.04", "chisel-versions": ["v1.0.0","main"]}, | |
| {"ref": "ubuntu-22.04", "chisel-versions": ["v1.0.0","main"]}, | |
| {"ref": "ubuntu-24.04", "chisel-versions": ["v1.0.0","main"]}, | |
| {"ref": "ubuntu-24.10", "chisel-versions": ["v1.0.0","main"]}, | |
| {"ref": "ubuntu-25.04", "chisel-versions": ["v1.0.0","main"]} | |
| ] | |
| SPLITS: "20" | |
| jobs: | |
| prepare-install: | |
| runs-on: ubuntu-latest | |
| name: "Prepare to install" | |
| outputs: | |
| install-all: ${{ steps.set-output.outputs.install_all }} | |
| matrix: ${{ steps.set-output.outputs.matrix }} | |
| checkout-main-ref: ${{ steps.set-main-ref.outputs.checkout_main_ref }} | |
| steps: | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Set reference to main branch | |
| id: set-main-ref | |
| run: | | |
| if [[ "${{ github.base_ref }}" == "main" ]]; then | |
| # For PRs to main, use the updated files. | |
| # Leaving checkout_main_ref unset will checkout the PR head_ref. | |
| echo "checkout_main_ref=" >> $GITHUB_OUTPUT | |
| else | |
| echo "checkout_main_ref=main" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Checkout main branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.set-main-ref.outputs.checkout_main_ref }} | |
| - name: Set output | |
| id: set-output | |
| run: | | |
| set -ex | |
| ln -s ".github/scripts/install-slices/version-matrix.py" version-matrix | |
| if [[ | |
| "${{ github.base_ref || github.ref_name }}" == "main" || | |
| "${{ github.event_name }}" == "schedule" | |
| ]]; then | |
| # When installing all the slices only use the main version of | |
| # chisel to avoid expensive CI jobs. We are testing that slices can | |
| # be installed together so using a single version of chisel is | |
| # enough. | |
| export RELEASES=$(echo "$RELEASES" | jq 'map(.["chisel-versions"] |= ["main"])') | |
| MATRIX=$(./version-matrix) | |
| echo "matrix={\"include\": $MATRIX}" >> $GITHUB_OUTPUT | |
| echo "install_all=true" >> $GITHUB_OUTPUT | |
| else | |
| # Filter the releases to only the affected branch, then swap the ref | |
| # such that the correct branch is tested. | |
| export RELEASES=$(echo "$RELEASES" | jq '[.[] | select(.ref == "${{ github.base_ref }}") | .ref = ""]') | |
| MATRIX=$(./version-matrix) | |
| echo "matrix={\"include\": $MATRIX}" >> $GITHUB_OUTPUT | |
| fi | |
| # The "install" job tests the slices by installing them. | |
| # It installs **all** slices if: | |
| # - chisel.yaml is changed | |
| # - any slice definition files are deleted | |
| # - github workflows (and related files) are changed | |
| # Otherwise, it installs only the slices from **added** and/or | |
| # **modified** slice definition files. | |
| # Please note that "change" is defined to encompass additions, | |
| # modifications and deletions here. | |
| install: | |
| runs-on: ubuntu-latest | |
| name: "Install" | |
| needs: prepare-install | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.prepare-install.outputs.matrix) }} | |
| env: | |
| install-all: ${{ needs.prepare-install.outputs.install-all }} | |
| main-branch-ref: ${{ needs.prepare-install.outputs.checkout-main-ref }} | |
| main-branch-path: files-from-main | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ matrix.ref }} | |
| - name: Check changed paths | |
| id: changed-paths | |
| if: env.install-all != 'true' | |
| uses: dorny/paths-filter@v3 | |
| with: | |
| # ref: https://github.com/marketplace/actions/paths-changes-filter | |
| filters: | | |
| install-all: | |
| - 'chisel.yaml' | |
| - deleted: 'slices/**/*.yaml' | |
| - '.github/**' | |
| slices: | |
| - added|modified: 'slices/**/*.yaml' | |
| # Space delimited list usable as command-line argument list in | |
| # Linux shell. If needed, it uses single or double quotes to | |
| # wrap filename with unsafe characters. | |
| list-files: shell | |
| - name: Setup Go environment | |
| uses: actions/setup-go@v5 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Checkout main branch | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.main-branch-ref }} | |
| path: ${{ env.main-branch-path }} | |
| - name: Install dependencies | |
| env: | |
| script-dir: "${{ env.main-branch-path }}/.github/scripts/install-slices" | |
| run: | | |
| set -ex | |
| # Install chisel | |
| go install "github.com/canonical/chisel/cmd/chisel@${{ matrix.chisel-version }}" | |
| # Install dependencies of the install_slices script | |
| sudo apt-get -y update | |
| sudo apt-get install -y $(cat "${{ env.script-dir }}/deb-requirements.txt") | |
| pip install -r "${{ env.script-dir }}/requirements.txt" | |
| # Configure the path of install_slices script | |
| ln -s "${{ env.script-dir }}/install_slices.py" install-slices | |
| ln -s "${{ env.script-dir }}/split.py" split | |
| # TODO: As we are installing the slices for every (ref, arch), when | |
| # installing all slices, we are also checking the existence of every | |
| # package for at least one architecture in a particular release. This | |
| # means that we are running the same check for every arch while running it | |
| # for just one arch would have been enough. We should revisit this in | |
| # future and propose improvements. | |
| # See also https://github.com/canonical/chisel-releases/pull/119#discussion_r1494785644 | |
| - name: Install slices | |
| run: | | |
| set -ex | |
| if [[ | |
| "${{ env.install-all }}" == "true" || | |
| "${{ steps.changed-paths.outputs.install-all }}" == "true" | |
| ]]; then | |
| # Install all slices in slices/ dir. | |
| # We need to enable globstar to use the ** patterns below. | |
| shopt -s globstar | |
| ./split "${{ matrix.id }}" "${{ env.SPLITS }}" slices/**/*.yaml | | |
| xargs ./install-slices \ | |
| --arch "${{ matrix.arch }}" \ | |
| --release ./ \ | |
| --ensure-existence \ | |
| --ignore-missing | |
| elif [[ "${{ steps.changed-paths.outputs.slices }}" == "true" ]]; then | |
| # Install slices from changed files. | |
| ./split "${{ matrix.id }}" "${{ env.SPLITS }}" \ | |
| ${{ steps.changed-paths.outputs.slices_files }} | | |
| xargs ./install-slices \ | |
| --arch "${{ matrix.arch }}" \ | |
| --release ./ \ | |
| --ensure-existence \ | |
| --ignore-missing \ | |
| fi |